diff --git a/.env.example b/.env.example index d99d26a2..bf8204f3 100644 --- a/.env.example +++ b/.env.example @@ -2,12 +2,15 @@ # These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 +LAB4_HTTP_PORT=8081 +LAB4_POSTGRES_PORT=5433 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis +REDIS_ADDR=redis:6379 CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -18,3 +21,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Set true to inspect candidates without deleting them. +LAB4_CLEANUP_DRY_RUN=false diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..a3655952 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,21 @@ +name: Lab4 application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Start disposable stack + run: | + sh scripts/init-local.sh + docker compose --env-file .env up -d --build --wait --wait-timeout 120 + - run: make test BASE_URL=http://127.0.0.1:8081 + - name: Remove disposable stack + if: always() + run: docker compose --env-file .env down --volumes diff --git a/.gitignore b/.gitignore index b5a71896..577093d0 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +# Transient local test runs; publish selected evidence under evidence/. +student-tests/results/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..978b3c0e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM golang:1.27.1-alpine AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd/api/ ./cmd/api/ +COPY cmd/attachment-cleanup/ ./cmd/attachment-cleanup/ +COPY internal/ ./internal/ + +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +RUN CGO_ENABLED=0 GOOS=linux go build -o /app/attachment-cleanup ./cmd/attachment-cleanup + + +FROM alpine:3.22 AS runtime + +RUN addgroup -S app && adduser -S app -G app + +WORKDIR /app + +USER app + +FROM runtime AS cleanup + +COPY --from=builder /app/attachment-cleanup ./attachment-cleanup +CMD ["./attachment-cleanup"] + +FROM runtime AS api + +COPY --from=builder /app/server ./server +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1 + +CMD ["./server"] diff --git a/Dockerfile.migrate b/Dockerfile.migrate new file mode 100644 index 00000000..67caf74d --- /dev/null +++ b/Dockerfile.migrate @@ -0,0 +1,19 @@ +FROM golang:1.27.1-alpine AS builder + +WORKDIR /app + +COPY go.mod go.sum ./ +RUN go mod download + +COPY cmd/migrate/ ./cmd/migrate/ +COPY db/migrations/ ./db/migrations/ +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /app/migrate ./cmd/migrate + +FROM alpine:3.22 + +RUN addgroup -S app && adduser -S app -G app +WORKDIR /app +COPY --from=builder /app/migrate ./migrate + +USER app +ENTRYPOINT ["/app/migrate"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..00f4b6ed 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,85 @@ -# Отчёт — лабораторная 4 +# Лабораторная 4. Вложения в S3 и сверка хранилищ -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Ремез Олег Романович · БПИ2304 · цель «5» · трек 2, согласованность S3 и PostgreSQL** -## Запуск из чистого clone +Мессенджер принимает PNG и JPEG, связывает оригиналы с сообщениями и отдаёт их участникам чата через любую из двух API-реплик. Приватный RustFS хранит байты, PostgreSQL — метаданные и связи, Redis — сессии. Команда `attachment-cleanup` обнаруживает расхождения в обе стороны. Старые объекты без строки удаляются после сверки, а пропавший оригинал восстанавливается из копии с сохранением ID и связей. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Запуск -## Архитектура и решения +Требуются Docker с Compose и OpenSSL. Скрипт подготовки создаёт локальные `.env` и `secrets/` при их отсутствии. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```sh +sh scripts/init-local.sh +docker compose --env-file .env up -d --build --wait +make check +make test BASE_URL=http://127.0.0.1:8081 +``` -## Доказательства критериев +API доступен через HAProxy на `127.0.0.1:8081`. Перезапуск стенда с сохранением volumes выполняется через `sh scripts/restart.sh`. Остановка выполняется через `docker compose --env-file .env down`. Для экспериментов используйте отдельный Compose-проект с собственными портами и синтетическими данными. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Как работает загрузка и периодическая уборка -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Обработчик принимает один PNG/JPEG до 10 МиБ и 40 млн пикселей. Тело копируется частями во временный файл на `tmpfs` с лимитом 32 МиБ на контейнер. Код закрывает дескриптор при завершении запроса, SHA-256 вычисляется по оригиналу, а S3 получает файл потоком. Каждая API-реплика обрабатывает одну загрузку одновременно. Запрос, который ждёт слот дольше секунды, получает `429` с `Retry-After: 1`. Проверка изображения декодирует его пиксели, поэтому ограничение по пикселям и слоту существенно для памяти. -## Эксперименты и отказы +API возвращает `201 ready` после записи оригинала в S3 и COMMIT в PostgreSQL. При обрыве между ними объект может остаться без метаданных. Уборка просматривает `attachments/` каждые 5 минут, проверяет только объекты старше 10 минут по S3 `LastModified` и удаляет их после успешного подтверждения отсутствия строки. Загрузка, уборка и восстановление используют одну транзакционную блокировку PostgreSQL по ключу. Занятый ключ переносится на следующий проход. Строка в `attachments` защищает оригинал с момента загрузки, в том числе до его привязки к сообщению. Ошибка хранилища прерывает проход. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Трек 2. Сверка в обе стороны -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Гипотеза.** По старым ключам можно находить два вида расхождений без загрузки байтов при обходе. Удаление неподтверждённого объекта и восстановление проверенного оригинала должны сохранять исправные вложения. Повторная сверка после исправления должна показать ноль расхождений. При росте числа исправных пар растут число проверок и время обхода. -## Ограничения и следующие шаги +| Направление | Проверка | Действие | +| --- | --- | --- | +| S3 → PostgreSQL | Постраничный LIST, фильтр по возрасту, EXISTS под блокировкой | В dry-run только посчитать кандидата. В apply удалить старый объект при успешном EXISTS, вернувшем `false` | +| PostgreSQL → S3 | Страницы по уникальному `object_key`, HEAD для подходящих по возрасту строк | Показать число пропаж и до 32 ID. Для выбранного ID принять копию, проверить размер и SHA-256, перечитать метаданные под блокировкой и выполнить условный PUT | -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Для автоматической уборки раз в 5 минут используется направление S3 → PostgreSQL. Полный двусторонний проход запускается явно и печатает JSON со счётчиками, длительностью и признаком завершения. Dry-run включён по умолчанию у разовой команды. Скан пропаж различает `404` от HEAD и сбой доступа. При сетевой ошибке или `403` проход завершается с ошибкой. Проверка существования бакета выполняется до HEAD объектов. [S3 HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) + +```sh +# Двусторонний dry-run +docker compose --env-file .env run --rm --no-deps -T attachment-cleanup \ + /app/attachment-cleanup -reconcile + +# Двусторонняя сверка с удалением старых объектов без строки +docker compose --env-file .env run --rm --no-deps -T attachment-cleanup \ + /app/attachment-cleanup -reconcile -dry-run=false + +# Dry-run восстановления одного пропавшего оригинала из проверяемой копии +docker compose --env-file .env run --rm --no-deps -T \ + -v "$PWD/original.png:/backup:ro" attachment-cleanup \ + /app/attachment-cleanup -restore-id UUID -backup-file /backup +``` + +Для восстановления добавьте `-dry-run=false` к последней команде. Запись отправляется с `If-None-Match: *`: если ключ уже существует, приложение его сохраняет, а S3 отклоняет конфликтующую запись, появившуюся после HEAD. При повторе с тем же оригиналом команда возвращает `already_present`. Источник резервных копий выбирает оператор. [Условная запись S3](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) + +### Эксперимент и результаты + +На отдельном стенде через API загружены четыре PNG. Затем у одного файла удалена строка PostgreSQL, у другого — объект S3. Пропавший оригинал был привязан к сообщению. После истечения тестового grace в 4 секунды dry-run нашёл по одному расхождению и оставил оба состояния без изменений. Копия с неверным SHA-256 отклонена. Верная копия успешно восстановлена, повтор вернул `already_present`. Прямой условный PUT по уже занятому ключу получил `412`. Проход apply удалил только старый объект без строки. Свежий объект и оба исправных файла, со связью и без неё, сохранились. + +После исправлений повторная сверка нашла ноль расхождений. У восстановленного файла совпали ID, метаданные, SHA-256 и ссылка в истории сообщений. Посторонний получил `404`. При имитации отказа S3 проход завершился с ошибкой, счётчик пропавших объектов остался нулевым. + +Затем добавлялись синтетические пары ключ/строка с однобайтовыми объектами. Для каждой выборки выполнены три dry-run без сторонней пользовательской нагрузки. В таблице указана длительность внутри процесса, отдельно от старта контейнера. Число HEAD и число проверок EXISTS в каждом проходе равны числу пар. + +| Пар | Страниц S3 / БД | Три прохода, мс | Медиана, мс | +| ---: | :--- | :--- | ---: | +| 33 | 1 / 1 | 82, 100, 32 | 82 | +| 233 | 1 / 2 | 227, 132, 147 | 147 | +| 1053 | 2 / 6 | 629, 587, 3070 | 629 | + +Объём запросов растёт вместе с числом ключей. Наибольший профиль пересёк границу страницы в обоих хранилищах. Разброс 587–3070 мс показывает, что три прогона на одном Docker-хосте дают ориентир, а предельную производительность надо измерять отдельно. [Исходные измерения](evidence/lab4-grade5/reconcile.json) · [Сценарий](student-tests/lab4/reconcile_experiment.py) · [Методика запуска](evidence/lab4-grade5/README.md) + +## Проверки уровня 4 + +| Проверка | Наблюдение | +| --- | --- | +| Сборка, миграции и совместимость API | Чистый Compose запускает миграции 1–6, RustFS и обе API. Повторная генерация sqlc совпала с кодом в ветке. Публичный smoke и 54 группы Go-тестов прошли на выделенном PostgreSQL и Redis без пропусков, включая штатные перезапуски. [Запуск и результаты](evidence/lab4-grade5/README.md) | +| Отказ API и перезапуск | Другая реплика отдала оригинал с тем же SHA-256. Перезапуск сохранил данные и сессию. Мигратор и bootstrap при штатном перезапуске не запустились повторно. [Переключение файла](evidence/lab4-final/file-failover.json) · [До](evidence/lab4-final/restart-before.txt) · [После](evidence/lab4-final/restart-after.txt) | +| Ошибки и приватность | Размер и число пикселей сверх лимита — `413`, повреждённое изображение и подмена MIME — `415`, чужой файл и привязка из другого чата — `404`. Анонимные GET/LIST S3 — `403`, ключ API ограничен бакетом, консоль закрыта. После повторного bootstrap объект сохранился. [HTTP](evidence/lab4-final/attachment-http.json) · [Пиксели](evidence/lab4-final/pixel-limit.json) · [Политика](evidence/lab4-final/s3-security.jsonl) | +| Отказ S3 и обрыв загрузки | У API `live=200`, `ready=503`, загрузка — `503`, новые метаданные не появились. После восстановления файл совпал по SHA-256. При обрыве HTTP, SIGKILL после PUT и ошибке INSERT после PUT уборка прошла; подтверждённая строка после потери ответа сохранилась. [Отказ S3](evidence/lab4-final/s3-failure.json) · [Сценарии уборки](evidence/lab4-grade5/compensation.json) | + +В двух прогонах по 12 одновременно запущенных загрузок лимит RAM составлял 512 МиБ на API. Обычный PNG 320 × 200 дал 12 ответов `201` с пиками 78,44 и 76,95 МиБ. При PNG/JPEG 6000 × 6000 четыре запроса получили `201`, восемь — `429`. Пики 280,59 и 410,29 МиБ включают процесс и `tmpfs`. OOM и перезапусков в обоих прогонах не было. [Обычный профиль](evidence/lab4-final/memory-normal.json) · [Тяжёлый профиль](evidence/lab4-final/memory-heavy.json) · [Методика](evidence/lab4-final/README.md) + +## Границы и источники + +Восстановление требует оригинальных байтов из резервной копии. HEAD подтверждает наличие объекта на момент запроса. Проверка целостности всех существующих объектов, восстановление потерянной строки БД и резервное копирование выходят за рамки трека. При ошибке или превышении двухминутного лимита проход завершится как неполный; уже выполненные удаления сохраняются. Сверка работает по текущим страницам, поэтому новые данные могут попасть в следующий проход. Стенд проверен на одном Docker-хосте, PostgreSQL, Redis и RustFS представлены по одному экземпляру. + +Gitea Actions для форка и базового репозитория отключены. Проверки выполнены локально. Публичные контракты и тесты преподавателя сохранены. Использован ИИ для проектирования, реализации части Go и Compose, тестовой оснастки и оформления отчёта. [Блокировки PostgreSQL](https://www.postgresql.org/docs/18/explicit-locking.html#ADVISORY-LOCKS) · [RustFS credentials](https://docs.rustfs.com/en/operations/credentials) diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 00000000..4ddc0c3d --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "errors" + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository/postgres" + "highload/internal/repository/redis" + "highload/internal/service" + "highload/internal/storage/s3" + httpdeliver "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" +) + +func main() { + + cfg, err := config.Load() + + if err != nil { + slog.Error("invalid configuration", "error", err) + os.Exit(1) + } + + sessionStore := cfg.Session.Store + + if sessionStore != "redis" { + slog.Error("session store is "+sessionStore+"must redis", "error", nil) + os.Exit(1) + } + + databaseURL := cfg.Database.PostgresURL + pool, err := database.NewPostgres(context.Background(), databaseURL) + + if err != nil { + slog.Error("postgres error", "error", err) + os.Exit(1) + } + defer pool.Close() + if os.Getenv("PG_POOL_DIAGNOSTICS") == "1" { + diagnosticsCtx, stopDiagnostics := context.WithCancel(context.Background()) + defer stopDiagnostics() + go logPoolStats(diagnosticsCtx, pool) + } + + queries := sqlc.New(pool) + + redisAddr := cfg.Database.RedisAddr + + redisClient, err := database.NewRedis(context.Background(), redisAddr) + + if err != nil { + slog.Error("redis error", "error", err) + os.Exit(1) + } + + defer redisClient.Close() + + instanceID, err := os.Hostname() + if err != nil || instanceID == "" { + slog.Error("cannot get instance ID", "error", err) + os.Exit(1) + } + + s3Storage, err := s3.NewStorage(context.Background(), &cfg.S3) + + if err != nil { + slog.Error("cannot create new storage", "error", err) + os.Exit(1) + } + + userRepo := postgres.NewUserRepository(queries) + chatRepo := postgres.NewChatRepository(queries, pool) + messageRepo := postgres.NewMessageRepository(queries, pool) + sessionRepo := redis.NewSessionRepository(redisClient) + attachmentRepo := postgres.NewAttachmentRepository(queries, pool) + + authService := service.NewAuthService(userRepo, sessionRepo, cfg.Session.TTL) + userService := service.NewUserService(userRepo) + chatService := service.NewChatService(chatRepo, userRepo, userService) + messageService := service.NewMessageService(messageRepo, chatService, userService) + attachmentService := service.NewAttachmentService(attachmentRepo, chatService, s3Storage) + + healthHandler := handler.NewHealthHandler(pool, redisClient, s3Storage) + authHandler := handler.NewAuthHandler(authService) + userHandler := handler.NewUserHandler(userService) + chatHandler := handler.NewChatHandler(chatService) + messageHandler := handler.NewMessageHandler(messageService) + attachmentHandler := handler.NewAttachmentHandler(attachmentService) + + mid := middleware.NewMiddleware(authService, instanceID) + router := httpdeliver.NewRouter(mid, authHandler, healthHandler, chatHandler, userHandler, messageHandler, attachmentHandler) + + server := &http.Server{ + Addr: net.JoinHostPort(cfg.HTTP.Host, cfg.HTTP.Port), + Handler: router.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 10 * time.Second, + IdleTimeout: 30 * time.Second, + } + + listener, err := net.Listen("tcp", server.Addr) + if err != nil { + slog.Error("server failed to start", "error", err) + return + } + + slog.Info("server started", "addr", server.Addr) + + serverErr := make(chan error, 1) + go func() { + err := server.Serve(listener) + if err != nil && !errors.Is(err, http.ErrServerClosed) { + serverErr <- err + slog.Error("server failed", "error", err) + } + }() + + ctx, stop := signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGTERM, + ) + defer stop() + + select { + case <-ctx.Done(): + slog.Info("shutdown signal received") + + case err := <-serverErr: + slog.Error("server failed", "error", err) + return + } + + shutdownCtx, cancel := context.WithTimeout( + context.Background(), + 15*time.Second, + ) + defer cancel() + + if err := server.Shutdown(shutdownCtx); err != nil { + slog.Error("shutdown failed", "error", err) + } + +} diff --git a/cmd/api/poolstats.go b/cmd/api/poolstats.go new file mode 100644 index 00000000..b57a70c6 --- /dev/null +++ b/cmd/api/poolstats.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + "log/slog" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// logPoolStats is opt-in diagnostics for the isolated scaling experiment. +// It contains only aggregate counters, never connection strings or SQL values. +func logPoolStats(ctx context.Context, pool *pgxpool.Pool) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s := pool.Stat() + slog.Info("pg_pool_stats", "max_conns", s.MaxConns(), + "acquired", s.AcquiredConns(), "idle", s.IdleConns(), + "acquire_count", s.AcquireCount(), "empty_count", s.EmptyAcquireCount(), + "empty_wait_ns", s.EmptyAcquireWaitTime().Nanoseconds(), + "canceled_count", s.CanceledAcquireCount()) + } + } +} diff --git a/cmd/attachment-cleanup/main.go b/cmd/attachment-cleanup/main.go new file mode 100644 index 00000000..79f791f6 --- /dev/null +++ b/cmd/attachment-cleanup/main.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "highload/internal/config" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage/s3" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +type cleanupConfig struct { + once bool + dryRun bool + gracePeriod time.Duration + interval time.Duration + passTimeout time.Duration + reconcile bool + restoreID string + backupFile string +} + +func main() { + var cfg cleanupConfig + + flag.BoolVar(&cfg.once, "once", false, "Run once and exit") + flag.BoolVar(&cfg.dryRun, "dry-run", true, "Do not actually delete files") + flag.DurationVar(&cfg.gracePeriod, "grace-period", 10*time.Minute, "Grace period for old files") + flag.DurationVar(&cfg.interval, "interval", 5*time.Minute, "Interval between runs") + flag.DurationVar(&cfg.passTimeout, "pass-timeout", 2*time.Minute, "Timeout for a single pass") + flag.BoolVar(&cfg.reconcile, "reconcile", false, "Scan both stores once and print JSON observations") + flag.StringVar(&cfg.restoreID, "restore-id", "", "Restore the missing original for this attachment ID") + flag.StringVar(&cfg.backupFile, "backup-file", "", "Path to the original backup used for restore") + flag.Parse() + + if cfg.gracePeriod <= 0 || cfg.interval <= 0 || cfg.passTimeout <= 0 { + slog.Error("all duration flags must be positive") + os.Exit(1) + } + if (cfg.restoreID == "") != (cfg.backupFile == "") || (cfg.reconcile && cfg.restoreID != "") { + slog.Error("use -reconcile alone, or supply both -restore-id and -backup-file") + os.Exit(1) + } + if err := run(cfg); err != nil { + slog.Error("attachment cleanup failed", "error", err) + os.Exit(1) + } +} + +func run(cfg cleanupConfig) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + databaseURL, s3Config, err := config.LoadAttachmentCleanup() + if err != nil { + return fmt.Errorf("load cleanup configuration: %w", err) + } + + startupCtx, cancelStartup := context.WithTimeout(ctx, 10*time.Second) + defer cancelStartup() + + pool, err := database.NewPostgres(startupCtx, databaseURL) + if err != nil { + var configErr *pgconn.ParseConfigError + if errors.As(err, &configErr) { + return errors.New("invalid PostgreSQL connection string") + } + return fmt.Errorf("connect to PostgreSQL: %w", err) + } + defer pool.Close() + + s3Storage, err := s3.NewStorage(startupCtx, &s3Config) + if err != nil { + return fmt.Errorf("create S3 client: %w", err) + } + cancelStartup() + + attachmentRepo := postgres.NewAttachmentRepository(sqlc.New(pool), pool) + options := &service.AttachmentCleanupOptions{ + GracePeriod: cfg.gracePeriod, + DryRun: cfg.dryRun, + ListTimeout: 5 * time.Second, + } + attachmentCleanupService := service.NewAttachmentCleanupService(attachmentRepo, s3Storage, options) + + if cfg.reconcile || cfg.restoreID != "" { + passCtx, cancel := context.WithTimeout(ctx, cfg.passTimeout) + defer cancel() + reconciler := service.NewAttachmentReconciler(attachmentRepo, s3Storage, *options) + start := time.Now() + report := struct { + DryRun bool `json:"dry_run"` + DurationMS int64 `json:"duration_ms"` + Complete bool `json:"complete"` + Orphans *service.OrphanReport `json:"orphan_scan,omitempty"` + Missing *service.MissingReport `json:"missing_scan,omitempty"` + Restore string `json:"restore,omitempty"` + }{DryRun: cfg.dryRun} + var actionErr error + if cfg.reconcile { + missing, err := reconciler.ScanMissing(passCtx) + report.Missing = &missing + actionErr = err + if err == nil { + orphans, err := attachmentCleanupService.ScanOrphans(passCtx) + report.Orphans = &orphans + actionErr = err + } + } else { + file, err := os.Open(cfg.backupFile) + if err != nil { + return fmt.Errorf("open backup original: %w", err) + } + defer file.Close() + report.Restore, actionErr = reconciler.Restore(passCtx, cfg.restoreID, file) + } + report.DurationMS = time.Since(start).Milliseconds() + report.Complete = actionErr == nil + return errors.Join(actionErr, json.NewEncoder(os.Stdout).Encode(report)) + } + + runPass := func() error { + passCtx, cancelPass := context.WithTimeout(ctx, cfg.passTimeout) + defer cancelPass() + return attachmentCleanupService.RunOnce(passCtx) + } + + if cfg.once { + return runPass() + } + + if err := runPass(); err != nil && ctx.Err() == nil { + slog.Error("attachment cleanup pass failed", "error", err) + } + + ticker := time.NewTicker(cfg.interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + if ctx.Err() != nil { + return nil + } + if err := runPass(); err != nil && ctx.Err() == nil { + slog.Error("attachment cleanup pass failed", "error", err) + } + } + } +} diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go new file mode 100644 index 00000000..cbe59e16 --- /dev/null +++ b/cmd/migrate/main.go @@ -0,0 +1,152 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "flag" + "log/slog" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "highload/db/migrations" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/pressly/goose/v3/lock" +) + +// Migration errors retain their cause for classification, but deliberately do +// not print connection strings, PostgreSQL DETAIL fields or raw SQL. +type migrationError struct { + stage string + cause error +} + +func (e *migrationError) Error() string { return e.stage } +func (e *migrationError) Unwrap() error { return e.cause } + +type boundedSessionLocker struct { + lock.SessionLocker +} + +func (l boundedSessionLocker) SessionUnlock(ctx context.Context, conn *sql.Conn) error { + // Goose unlocks with a detached context; bound this cleanup separately. + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + return l.SessionLocker.SessionUnlock(ctx, conn) +} + +func main() { + command := flag.String("command", "up", "migration action: up or status") + timeout := flag.Duration("timeout", 2*time.Minute, "total migration/connection/lock budget") + flag.Parse() + logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if flag.NArg() != 0 || (*command != "up" && *command != "status") || *timeout < time.Second { + logger.Error("invalid migration arguments", "allowed_commands", "up,status", + "minimum_timeout", "1s") + os.Exit(1) + } + + signalCtx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancel := context.WithTimeout(signalCtx, *timeout) + defer cancel() + if err := run(ctx, *command, *timeout, logger); err != nil { + fields := []any{"stage", err.Error(), "timeout", errors.Is(err, context.DeadlineExceeded), + "cancelled", errors.Is(err, context.Canceled)} + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + fields = append(fields, "sqlstate", pgErr.Code) + } + var partial *goose.PartialError + if errors.As(err, &partial) && partial.Failed != nil && partial.Failed.Source != nil { + fields = append(fields, "migration", partial.Failed.Source.Path, + "version", partial.Failed.Source.Version) + } + logger.Error("migration failed", fields...) + os.Exit(1) + } +} + +func run(ctx context.Context, command string, timeout time.Duration, logger *slog.Logger) error { + fail := func(stage string, err error) error { + return &migrationError{stage: stage, cause: err} + } + path := os.Getenv("DATABASE_URL_FILE") + if path == "" { + path = "/run/secrets/database_url" + } + data, err := os.ReadFile(path) + if err != nil { + return fail("read database secret file", err) + } + url := strings.TrimSpace(string(data)) + if url == "" { + return fail("database secret file is empty", errors.New("empty database URL")) + } + cfg, err := pgx.ParseConfig(url) + if err != nil { + return fail("parse database secret", err) + } + cfg.ConnectTimeout = 5 * time.Second + cfg.RuntimeParams["application_name"] = "highload-migrate" + cfg.RuntimeParams["lock_timeout"] = "10000" + cfg.RuntimeParams["statement_timeout"] = strconv.FormatInt(min(timeout.Milliseconds(), 2147483647), 10) + db := stdlib.OpenDB(*cfg) + db.SetMaxOpenConns(2) + db.SetMaxIdleConns(2) + db.SetConnMaxLifetime(5 * time.Minute) + defer db.Close() + if err := db.PingContext(ctx); err != nil { + return fail("connect to PostgreSQL", err) + } + + locker, err := lock.NewPostgresSessionLocker( + lock.WithLockTimeout(1, 120), + lock.WithUnlockTimeout(1, 5), + ) + if err != nil { + return fail("configure migration lock", err) + } + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files, + goose.WithSessionLocker(boundedSessionLocker{locker}), + goose.WithDisableGlobalRegistry(true), + goose.WithSlog(logger), + // Goose verbose logging includes SQL statements. Log only selected + // migration metadata ourselves, even when a migration fails. + goose.WithVerbose(false), + ) + if err != nil { + return fail("load embedded migrations", err) + } + if command == "status" { + statuses, err := provider.Status(ctx) + if err != nil { + return fail("read migration status", err) + } + for _, status := range statuses { + logger.Info("migration status", "file", status.Source.Path, + "version", status.Source.Version, "state", status.State) + } + return nil + } + started := time.Now() + results, err := provider.Up(ctx) + if err != nil { + return fail("apply pending migrations", err) + } + for _, result := range results { + logger.Info("migration applied", "file", result.Source.Path, + "version", result.Source.Version, "duration_seconds", result.Duration.Seconds()) + } + logger.Info("migrations completed", "applied", len(results), + "duration_seconds", time.Since(started).Seconds()) + return nil +} diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go new file mode 100644 index 00000000..745a8605 --- /dev/null +++ b/cmd/migrate/main_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "database/sql" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "highload/db/migrations" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" +) + +func TestConnectionSecretErrorsAreSafe(t *testing.T) { + for _, test := range []struct { + name string + content string + missing bool + }{ + {name: "missing", missing: true}, + {name: "empty", content: " \n\t"}, + {name: "malformed", content: "postgres://synthetic:fixture-password@localhost/%zz"}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "database_url") + if !test.missing { + if err := os.WriteFile(path, []byte(test.content), 0600); err != nil { + t.Fatal("cannot create test secret") + } + } + t.Setenv("DATABASE_URL_FILE", path) + logger := slog.New(slog.NewJSONHandler(io.Discard, nil)) + err := run(context.Background(), "up", time.Second, logger) + if err == nil { + t.Fatal("invalid connection secret was accepted") + } + if strings.Contains(err.Error(), "fixture-password") || + strings.Contains(err.Error(), "postgres://") { + t.Fatal("migration error exposed a connection secret") + } + var classified *migrationError + if !errors.As(err, &classified) || classified.Unwrap() == nil { + t.Fatal("migration error lost its classification/cause") + } + }) + } +} + +func TestEmbeddedMigrationsAreVersionedAndCollectable(t *testing.T) { + cfg, err := pgx.ParseConfig("postgres://synthetic@localhost/synthetic?sslmode=disable") + if err != nil { + t.Fatal("cannot configure test driver") + } + db := stdlib.OpenDB(*cfg) + defer db.Close() + // NewProvider collects sources without connecting to PostgreSQL. + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files) + if err != nil { + t.Fatal("embedded migration names/versions cannot be collected by Goose") + } + sources := provider.ListSources() + if len(sources) == 0 { + t.Fatal("migrator build contains no SQL") + } + var previous int64 + for _, source := range sources { + if source.Version <= previous { + t.Fatal("embedded versions are not unique and increasing") + } + previous = source.Version + data, err := migrations.Files.ReadFile(source.Path) + if err != nil || !strings.Contains(string(data), "-- +goose Up") { + t.Fatal("embedded migration is unavailable or has no Up section") + } + } +} + +type deadlineProbe struct { + deadline time.Time + cancel error +} + +func (*deadlineProbe) SessionLock(context.Context, *sql.Conn) error { return nil } +func (p *deadlineProbe) SessionUnlock(ctx context.Context, _ *sql.Conn) error { + p.deadline, _ = ctx.Deadline() + p.cancel = ctx.Err() + return nil +} + +func TestUnlockHasBudgetEvenWithDetachedContext(t *testing.T) { + original, cancel := context.WithCancel(context.Background()) + cancel() + probe := &deadlineProbe{} + locker := boundedSessionLocker{probe} + started := time.Now() + if err := locker.SessionUnlock(context.WithoutCancel(original), nil); err != nil { + t.Fatal("unlock delegation failed") + } + if probe.cancel != nil || probe.deadline.IsZero() || + probe.deadline.Before(started) || probe.deadline.After(started.Add(6*time.Second)) { + t.Fatal("detached cleanup did not receive a bounded, live context") + } +} diff --git a/cmd/migrate/postgres_test.go b/cmd/migrate/postgres_test.go new file mode 100644 index 00000000..08aad862 --- /dev/null +++ b/cmd/migrate/postgres_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "log/slog" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/stdlib" +) + +// These tests need an explicitly supplied disposable PostgreSQL database. +// Each test creates and drops only its own randomly named schema. +func migrationPostgresFixture(t *testing.T) (*sql.DB, context.Context) { + t.Helper() + connectionURL := os.Getenv("MIGRATE_TEST_DATABASE_URL") + if connectionURL == "" { + t.Skip("set MIGRATE_TEST_DATABASE_URL to run PostgreSQL integration tests") + } + cfg, err := pgx.ParseConfig(connectionURL) + if err != nil { + t.Fatal("invalid test database configuration") + } + cfg.ConnectTimeout = 5 * time.Second + admin := stdlib.OpenDB(*cfg) + t.Cleanup(func() { admin.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + schema := "migration_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := admin.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal("cannot create isolated migration test schema") + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := admin.ExecContext(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error("cannot remove isolated migration test schema") + } + }) + cfg.RuntimeParams["search_path"] = schema + // ConnString retains the original parsed URL; changing RuntimeParams alone + // would not pass search_path to the separate migrator connection. + scopedURL, err := url.Parse(connectionURL) + if err != nil || (scopedURL.Scheme != "postgres" && scopedURL.Scheme != "postgresql") { + t.Fatal("MIGRATE_TEST_DATABASE_URL must be a PostgreSQL URL") + } + query := scopedURL.Query() + query.Set("search_path", schema) + scopedURL.RawQuery = query.Encode() + path := filepath.Join(t.TempDir(), "database_url") + if err := os.WriteFile(path, []byte(scopedURL.String()), 0600); err != nil { + t.Fatal("cannot write test connection secret") + } + t.Setenv("DATABASE_URL_FILE", path) + db := stdlib.OpenDB(*cfg) + t.Cleanup(func() { db.Close() }) + return db, ctx +} + +func migrationLogRecords(t *testing.T, output *bytes.Buffer) []map[string]any { + t.Helper() + var records []map[string]any + for _, line := range bytes.Split(bytes.TrimSpace(output.Bytes()), []byte("\n")) { + if len(line) == 0 { + continue + } + var record map[string]any + if err := json.Unmarshal(line, &record); err != nil { + t.Fatal("migrator emitted a non-JSON log record") + } + for key := range record { + switch key { + case "time", "level", "msg", "file", "version", "state", "applied", "duration_seconds": + default: + t.Fatalf("unexpected log field %q; SQL or connection details must not be logged", key) + } + } + if strings.Contains(string(line), "CREATE TABLE") || strings.Contains(string(line), "postgres://") { + t.Fatal("migration log contains SQL or a connection URL") + } + records = append(records, record) + } + return records +} + +func completedMigrationCount(t *testing.T, output *bytes.Buffer) int { + t.Helper() + for _, record := range migrationLogRecords(t, output) { + if record["msg"] == "migrations completed" { + count, ok := record["applied"].(float64) + if !ok { + t.Fatal("migration completion log has no numeric applied count") + } + return int(count) + } + } + t.Fatal("successful migration run has no completion log") + return 0 +} + +const lab4MigrationCount = 6 + +func TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + if err := run(ctx, "up", time.Minute, logger); err != nil { + t.Fatal("migration of empty schema failed") + } + if count := completedMigrationCount(t, &output); count != lab4MigrationCount { + t.Fatalf("empty schema: applied %d migrations, want %d", count, lab4MigrationCount) + } + if _, err := db.ExecContext(ctx, "INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3)", + "migration_fixture", "Migration fixture", "synthetic-test-only-hash"); err != nil { + t.Fatal("cannot insert persistent migration fixture") + } + output.Reset() + if err := run(ctx, "up", time.Minute, logger); err != nil { + t.Fatal("repeat migration failed") + } + if count := completedMigrationCount(t, &output); count != 0 { + t.Fatalf("repeat run: applied %d migrations, want 0", count) + } + var users, versions int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM users WHERE username = $1", "migration_fixture").Scan(&users); err != nil || users != 1 { + t.Fatal("repeat migration lost or duplicated existing data") + } + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != lab4MigrationCount { + t.Fatal("repeat migration changed version history") + } + output.Reset() + if err := run(ctx, "status", time.Minute, logger); err != nil { + t.Fatal("migration status failed") + } + records := migrationLogRecords(t, &output) + if len(records) != lab4MigrationCount { + t.Fatal("status did not report all six migrations") + } + for _, record := range records { + if record["state"] != "applied" { + t.Fatal("status did not report applied migrations") + } + } +} + +func TestPostgresConcurrentMigratorsApplyEachVersionOnce(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + var outputs [2]bytes.Buffer + var runErrors [2]error + var workers sync.WaitGroup + start := make(chan struct{}) + for i := range outputs { + workers.Add(1) + go func(i int) { + defer workers.Done() + <-start + runErrors[i] = run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&outputs[i], nil))) + }(i) + } + close(start) + workers.Wait() + var applied int + for i, err := range runErrors { + if err != nil { + t.Fatal("concurrent migration run failed") + } + applied += completedMigrationCount(t, &outputs[i]) + } + if applied != lab4MigrationCount { + t.Fatalf("concurrent runners applied %d migrations in total, want %d", applied, lab4MigrationCount) + } + var versions, distinctVersions int + if err := db.QueryRowContext(ctx, "SELECT count(*), count(DISTINCT version_id) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions, &distinctVersions); err != nil || versions != lab4MigrationCount || distinctVersions != lab4MigrationCount { + t.Fatal("concurrent runners duplicated or lost version history") + } +} + +func TestPostgresMigrationFailureRollsBackAndPreservesCause(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + // The first migration creates users, then conflicts with this chats table. + if _, err := db.ExecContext(ctx, "CREATE TABLE chats (marker text)"); err != nil { + t.Fatal("cannot create conflicting test table") + } + var output bytes.Buffer + err := run(ctx, "up", time.Minute, slog.New(slog.NewJSONHandler(&output, nil))) + var pgErr *pgconn.PgError + if err == nil || err.Error() != "apply pending migrations" || !errors.As(err, &pgErr) || pgErr.Code != "42P07" { + t.Fatal("migration failure did not retain a safely reportable duplicate-table error") + } + migrationLogRecords(t, &output) + var usersAbsent bool + if err := db.QueryRowContext(ctx, "SELECT to_regclass('users') IS NULL").Scan(&usersAbsent); err != nil || !usersAbsent { + t.Fatal("failed migration left a partially applied users table") + } + var versions int + if err := db.QueryRowContext(ctx, "SELECT count(*) FROM goose_db_version WHERE version_id > 0 AND is_applied").Scan(&versions); err != nil || versions != 0 { + t.Fatal("failed migration was recorded as applied") + } +} diff --git a/cmd/migrate/roundtrip_test.go b/cmd/migrate/roundtrip_test.go new file mode 100644 index 00000000..d32d4ae1 --- /dev/null +++ b/cmd/migrate/roundtrip_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "context" + "database/sql" + "io" + "log/slog" + "testing" + + "github.com/pressly/goose/v3" + "highload/db/migrations" +) + +func sessionSchema(t *testing.T, db *sql.DB, ctx context.Context) []string { + t.Helper() + rows, err := db.QueryContext(ctx, ` + SELECT 'column:'||attname||':'||format_type(atttypid,atttypmod)||':'||attnotnull||':'|| + COALESCE(pg_get_expr(d.adbin,d.adrelid),'') + FROM pg_attribute a LEFT JOIN pg_attrdef d ON d.adrelid=a.attrelid AND d.adnum=a.attnum + WHERE a.attrelid='sessions'::regclass AND a.attnum>0 AND NOT a.attisdropped + UNION ALL SELECT 'constraint:'||conname||':'||pg_get_constraintdef(oid) + FROM pg_constraint WHERE conrelid='sessions'::regclass + UNION ALL SELECT 'index:'||indexname||':'||indexdef + FROM pg_indexes WHERE schemaname=current_schema() AND tablename='sessions' + ORDER BY 1`) + if err != nil { + t.Fatal("cannot inspect restored session columns/defaults/constraints/indexes") + } + defer rows.Close() + var result []string + for rows.Next() { + var s string + if rows.Scan(&s) != nil { + t.Fatal("schema scan failed") + } + result = append(result, s) + } + if rows.Err() != nil || len(result) < 10 { + t.Fatal("incomplete session structure snapshot") + } + return result +} + +func businessData(t *testing.T, db *sql.DB, ctx context.Context) []string { + t.Helper() + result := []string{} + for _, table := range []string{"users", "chats", "chat_members", "messages"} { + rows, err := db.QueryContext(ctx, "SELECT row_to_json(t)::text FROM "+table+" t ORDER BY row_to_json(t)::text") + if err != nil { + t.Fatal("business table disappeared during migration") + } + count := 0 + for rows.Next() { + var value string + if rows.Scan(&value) != nil { + t.Fatal("data scan failed") + } + result = append(result, table+value) + count++ + } + err = rows.Err() + rows.Close() + if err != nil || count == 0 { + t.Fatal("business fixture missing during migration") + } + } + return result +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func TestPostgresMigrationThreeDownTwoDownOneAndReapply(t *testing.T) { + db, ctx := migrationPostgresFixture(t) + provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files, + goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil)))) + if err != nil { + t.Fatal("cannot load actual embedded migrations") + } + if _, err = provider.UpTo(ctx, 1); err != nil { + t.Fatal("cannot apply version 1") + } + v1 := sessionSchema(t, db, ctx) + if _, err = db.ExecContext(ctx, ` + INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES + ('00000000-0000-4000-8000-000000000001','roundtrip_user','Roundtrip','synthetic-hash','2026-01-01T00:00:00Z'); + INSERT INTO chats(id,title,created_at) VALUES + ('00000000-0000-4000-8000-000000000002','Roundtrip','2026-01-01T00:00:00Z'); + INSERT INTO chat_members VALUES('00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001'); + INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES + ('00000000-0000-4000-8000-000000000003','00000000-0000-4000-8000-000000000002','00000000-0000-4000-8000-000000000001','Keep this message','2026-01-01T00:00:00Z'); + INSERT INTO sessions VALUES(decode(repeat('ab',32),'hex'),'00000000-0000-4000-8000-000000000001','2026-01-01T00:00:00Z','2026-01-02T00:00:00Z',NULL); + `); err != nil { + t.Fatal("cannot create migration preservation fixtures") + } + original := businessData(t, db, ctx) + if _, err = provider.UpTo(ctx, 2); err != nil { + t.Fatal("cannot apply version 2") + } + v2 := sessionSchema(t, db, ctx) + assertVersion := func(want int64) { + t.Helper() + got, e := provider.GetDBVersion(ctx) + if e != nil || got != want { + t.Fatalf("want migration version %d, got %d", want, got) + } + if !equalStrings(original, businessData(t, db, ctx)) { + t.Fatal("migration changed unrelated users/chats/members/messages") + } + } + assertAbsent := func() { + t.Helper() + var absent bool + if db.QueryRowContext(ctx, "SELECT to_regclass('sessions') IS NULL").Scan(&absent) != nil || !absent { + t.Fatal("version 3 must remove SQL sessions") + } + } + if _, err = provider.Up(ctx); err != nil { + t.Fatal("cannot migrate to 6") + } + assertVersion(6) + assertAbsent() + if _, err = provider.DownTo(ctx, 2); err != nil { + t.Fatal("6 -> 2 failed") + } + assertVersion(2) + if !equalStrings(v2, sessionSchema(t, db, ctx)) { + t.Fatal("3 -> 2 did not restore exact version-2 session columns, defaults, keys, FK, CHECK and indexes") + } + var count int + if db.QueryRowContext(ctx, "SELECT count(*) FROM sessions").Scan(&count) != nil || count != 0 { + t.Fatal("rollback must restore an empty sessions table; deleted credentials cannot be resurrected") + } + if _, err = db.ExecContext(ctx, `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`); err != nil { + t.Fatal("restored table/default/FK cannot accept a valid session") + } + for _, statement := range []string{ + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000099',now()+interval '1 hour')`, + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('ef',32),'hex'),'00000000-0000-4000-8000-000000000001',now()-interval '1 hour')`, + `INSERT INTO sessions(token_hash,user_id,expires_at) VALUES(decode(repeat('cd',32),'hex'),'00000000-0000-4000-8000-000000000001',now()+interval '1 hour')`, + } { + if _, err = db.ExecContext(ctx, statement); err == nil { + t.Fatal("restored table lost FK, expiry CHECK or primary key") + } + } + if _, err = provider.DownTo(ctx, 1); err != nil { + t.Fatal("2 -> 1 failed") + } + assertVersion(1) + if !equalStrings(v1, sessionSchema(t, db, ctx)) { + t.Fatal("2 -> 1 did not restore original sessions structure without created_at default") + } + if _, err = provider.Up(ctx); err != nil { + t.Fatal("1 -> 6 reapplication failed") + } + assertVersion(6) + assertAbsent() +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..51dc9c4e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,218 @@ +name: highload-lab4 + +services: + api: + build: + context: . + dockerfile: Dockerfile + scale: 2 + cpus: "2.0" + mem_limit: 512m + expose: + - "8080" + env_file: + - .env + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + read_only: true + tmpfs: + - /tmp:size=32m,mode=1777 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + attachment-cleanup: + build: + context: . + dockerfile: Dockerfile + target: cleanup + command: ["/app/attachment-cleanup", "-dry-run=${LAB4_CLEANUP_DRY_RUN:-false}"] + cpus: "0.5" + mem_limit: 128m + env_file: + - .env + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + restart: unless-stopped + stop_grace_period: 20s + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + lb: + image: haproxy:3.2.24-alpine + cpus: "1.0" + mem_limit: 256m + ports: + - "127.0.0.1:${LAB4_HTTP_PORT:-8081}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + restart: unless-stopped + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + migrate: + build: + context: . + dockerfile: Dockerfile.migrate + cpus: "1.0" + mem_limit: 256m + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + stop_grace_period: 20s + read_only: true + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + postgres: + image: postgres:18 + cpus: "2.0" + mem_limit: 512m + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + ports: + - "127.0.0.1:${LAB4_POSTGRES_PORT:-5433}:5432" + volumes: + - postgres_data:/var/lib/postgresql + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + stop_grace_period: 60s + + redis: + image: redis:7.4-alpine + cpus: "0.5" + mem_limit: 128m + restart: unless-stopped + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + s3: + image: rustfs/rustfs:1.0.4 + cpus: "1.0" + mem_limit: 512m + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ACCESS_KEY_FILE: /run/secrets/rustfs_root_access_key + RUSTFS_SECRET_KEY_FILE: /run/secrets/rustfs_root_secret_key + RUSTFS_CONSOLE_ENABLE: "false" + secrets: + - rustfs_root_access_key + - rustfs_root_secret_key + volumes: + - rustfs_data:/data + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 30s + restart: unless-stopped + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + + s3-init: + image: rustfs/rc:v0.1.30 + cpus: "0.5" + mem_limit: 128m + entrypoint: ["/bin/sh", "/bootstrap.sh"] + environment: + HOME: /tmp + S3_BUCKET: messenger-attachments + S3_REGION: us-east-1 + secrets: + - rustfs_root_access_key + - rustfs_root_secret_key + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/s3/bootstrap.sh:/bootstrap.sh:ro + depends_on: + s3: + condition: service_healthy + restart: "no" + read_only: true + tmpfs: + - /tmp:uid=100,gid=101,mode=0700 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + +secrets: + database_url: + file: ./secrets/database_url + postgres_password: + file: ./secrets/postgres_password + rustfs_root_access_key: + file: ./secrets/rustfs_root_access_key + rustfs_root_secret_key: + file: ./secrets/rustfs_root_secret_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + postgres_data: + redis_data: + rustfs_data: diff --git a/db/migrations/00001_init_schema.sql b/db/migrations/00001_init_schema.sql new file mode 100644 index 00000000..1edaef66 --- /dev/null +++ b/db/migrations/00001_init_schema.sql @@ -0,0 +1,79 @@ +-- +goose Up +CREATE TABLE users +( + id uuid PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + username VARCHAR(32) UNIQUE NOT NULL, + display_name VARCHAR(100) NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT users_username_format_check CHECK (username COLLATE "C" ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length_check CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_display_name_not_blank_check CHECK (char_length(btrim(display_name)) > 0), + CONSTRAINT users_password_hash_not_empty_check CHECK (char_length(password_hash) > 0) + +); + + + +CREATE TABLE chats +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + title VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + CONSTRAINT chats_title_length_check CHECK (char_length(title) BETWEEN 1 AND 100 ), + CONSTRAINT chats_title_not_blank_check CHECK (char_length(btrim(title)) > 0) + +); + +CREATE TABLE chat_members +( + chat_id UUID NOT NULL REFERENCES chats (id), + user_id UUID NOT NULL REFERENCES users (id), + + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX idx_user_chat_id ON chat_members (user_id, chat_id); + + +CREATE TABLE messages +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + + CONSTRAINT messages_text_length_check CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_text_not_blank_check CHECK (char_length(btrim(text)) > 0) +); + +CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id); + + +CREATE TABLE sessions +( + token_hash BYTEA PRIMARY KEY NOT NULL, + user_id UUID NOT NULL REFERENCES users (id), + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ NULL, + + CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at) +); + +CREATE INDEX idx_session_user_id ON sessions (user_id); + + +-- +goose Down + +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS chat_members; +DROP TABLE IF EXISTS chats; +DROP TABLE IF EXISTS users; diff --git a/db/migrations/00002_add_created_at_defaults.sql b/db/migrations/00002_add_created_at_defaults.sql new file mode 100644 index 00000000..e6c9b1df --- /dev/null +++ b/db/migrations/00002_add_created_at_defaults.sql @@ -0,0 +1,20 @@ +-- +goose Up +ALTER TABLE users + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE chats + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE messages + ALTER COLUMN created_at SET DEFAULT now(); +ALTER TABLE sessions + ALTER COLUMN created_at SET DEFAULT now(); + +-- +goose Down + +ALTER TABLE sessions + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE messages + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE chats + ALTER COLUMN created_at DROP DEFAULT; +ALTER TABLE users + ALTER COLUMN created_at DROP DEFAULT; diff --git a/db/migrations/00003_drop_session_table.sql b/db/migrations/00003_drop_session_table.sql new file mode 100644 index 00000000..e392cd89 --- /dev/null +++ b/db/migrations/00003_drop_session_table.sql @@ -0,0 +1,17 @@ +-- +goose Up +DROP TABLE IF EXISTS sessions; + +-- +goose Down +-- Restore the schema at version 2; deleted session data cannot be recovered. +CREATE TABLE sessions +( + token_hash BYTEA PRIMARY KEY NOT NULL, + user_id UUID NOT NULL REFERENCES users (id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ NULL, + + CONSTRAINT sessions_expiry_after_creation_check CHECK (expires_at > created_at) +); + +CREATE INDEX idx_session_user_id ON sessions (user_id); diff --git a/db/migrations/00004_message_idempotency.sql b/db/migrations/00004_message_idempotency.sql new file mode 100644 index 00000000..54fdfe2a --- /dev/null +++ b/db/migrations/00004_message_idempotency.sql @@ -0,0 +1,23 @@ +-- +goose Up + +CREATE TABLE message_idempotency +( + user_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + idempotency_key VARCHAR(128) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL DEFAULT (now() + interval '24 hours'), + + PRIMARY KEY (user_id, chat_id, idempotency_key), + UNIQUE (message_id), + + CONSTRAINT message_idempotency_key_format_check + CHECK (idempotency_key COLLATE "C" ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_expiry_check + CHECK (expires_at > created_at) +); + +-- +goose Down + +DROP TABLE IF EXISTS message_idempotency; diff --git a/db/migrations/00005_attachments.sql b/db/migrations/00005_attachments.sql new file mode 100644 index 00000000..0e8a790f --- /dev/null +++ b/db/migrations/00005_attachments.sql @@ -0,0 +1,54 @@ +-- +goose Up +CREATE TABLE attachments +( + id UUID PRIMARY KEY NOT NULL DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename text NOT NULL, + object_key text UNIQUE NOT NULL, + content_type VARCHAR(10) NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + + CONSTRAINT fk_attachments_chat_member FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id), + CONSTRAINT attachments_sha256_format_check CHECK (sha256 COLLATE "C" ~ '^[0-9a-f]{64}$'), + CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_filename_check CHECK ( + char_length(filename) BETWEEN 1 AND 255 + AND filename !~ '[/\\]' + ), + CONSTRAINT attachments_size_bytes_check CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_id_chat_unique UNIQUE (id, chat_id), + CONSTRAINT attachments_object_key_length_check CHECK (char_length(object_key) > 0) + +); + +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE messages_attachments +( + message_id UUID NOT NULL , + attachment_id UUID NOT NULL , + chat_id UUID NOT NULL, + position SMALLINT NOT NULL, + + + CONSTRAINT message_attachments_pk PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9), + + CONSTRAINT message_attachments_message_chat_fk + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id), + CONSTRAINT message_attachments_attachment_chat_fk + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) +); + + +-- +goose Down +DROP TABLE IF EXISTS messages_attachments; +DROP TABLE IF EXISTS attachments; +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_unique; diff --git a/db/migrations/00006_allow_attachment_only_messages.sql b/db/migrations/00006_allow_attachment_only_messages.sql new file mode 100644 index 00000000..75b967ba --- /dev/null +++ b/db/migrations/00006_allow_attachment_only_messages.sql @@ -0,0 +1,18 @@ +-- +goose Up +ALTER TABLE messages + DROP CONSTRAINT messages_text_length_check, + DROP CONSTRAINT messages_text_not_blank_check; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length_check + CHECK (char_length(text) <= 4000); + +-- +goose Down +ALTER TABLE messages + DROP CONSTRAINT messages_text_length_check; + +ALTER TABLE messages + ADD CONSTRAINT messages_text_length_check + CHECK (char_length(text) BETWEEN 1 AND 4000), + ADD CONSTRAINT messages_text_not_blank_check + CHECK (char_length(btrim(text)) > 0); \ No newline at end of file diff --git a/db/migrations/embed.go b/db/migrations/embed.go new file mode 100644 index 00000000..ee488c7d --- /dev/null +++ b/db/migrations/embed.go @@ -0,0 +1,9 @@ +package migrations + +import "embed" + +// Files contains the SQL shipped with this exact migrator build. +// Add schema changes as new numbered files; do not edit applied migrations. +// +//go:embed *.sql +var Files embed.FS diff --git a/db/queries/attachments.sql b/db/queries/attachments.sql new file mode 100644 index 00000000..1e09f0c2 --- /dev/null +++ b/db/queries/attachments.sql @@ -0,0 +1,41 @@ +-- name: CreateAttachment :one +INSERT INTO attachments + (chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) +VALUES ($1, $2, $3, $4, $5, $6, $7) +RETURNING *; + +-- name: GetAttachmentForUser :one +SELECT a.* +FROM attachments AS a +JOIN chat_members AS cm ON cm.chat_id = a.chat_id +WHERE a.id = sqlc.arg(attachment_id) + AND cm.user_id = sqlc.arg(user_id); + + +-- name: GetAttachmentsForMessages :many +SELECT message_id, attachment_id, position +FROM messages_attachments +WHERE message_id = ANY(sqlc.arg(message_ids)::uuid[]) +ORDER BY message_id, position; + + +-- name: AttachmentExistsByObjectKey :one +SELECT EXISTS ( + SELECT 1 FROM attachments + WHERE object_key = sqlc.arg(object_key) +); + +-- name: TryLockAttachmentObjectKey :one +SELECT pg_try_advisory_xact_lock( + hashtextextended(sqlc.arg(object_key)::text, 0) +)::boolean AS locked; + +-- name: ListAttachmentKeys :many +SELECT id, object_key, created_at +FROM attachments +WHERE object_key > sqlc.arg(after_key) +ORDER BY object_key +LIMIT sqlc.arg(page_size); + +-- name: GetAttachmentByID :one +SELECT * FROM attachments WHERE id = sqlc.arg(attachment_id); diff --git a/db/queries/chat_members.sql b/db/queries/chat_members.sql new file mode 100644 index 00000000..811fdd8b --- /dev/null +++ b/db/queries/chat_members.sql @@ -0,0 +1,3 @@ +-- name: CreateChatMember :exec +INSERT INTO chat_members(chat_id, user_id) +VALUES ($1, $2); diff --git a/db/queries/chats.sql b/db/queries/chats.sql new file mode 100644 index 00000000..2bfe8e0f --- /dev/null +++ b/db/queries/chats.sql @@ -0,0 +1,34 @@ +-- name: CreateChat :one +INSERT INTO chats (title) +VALUES ($1) +RETURNING id, title, created_at; + + +-- name: GetChatByID :one + +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE c.id = $1 +GROUP BY c.id; + + +-- name: GetChatsByUserID :many +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE EXISTS (SELECT 1 + FROM chat_members membership + WHERE membership.chat_id = c.id + AND membership.user_id = sqlc.arg(user_id)) + AND ( + sqlc.narg(after_created_at)::timestamptz IS NULL + OR (c.created_at, c.id) > + ( + sqlc.narg(after_created_at)::timestamptz, + sqlc.narg(after_id)::uuid + ) + ) +GROUP BY c.id +ORDER BY c.created_at ASC, c.id ASC +LIMIT sqlc.arg(limit_count); \ No newline at end of file diff --git a/db/queries/message_idempotence.sql b/db/queries/message_idempotence.sql new file mode 100644 index 00000000..ebea13de --- /dev/null +++ b/db/queries/message_idempotence.sql @@ -0,0 +1,13 @@ +-- name: InsertMessageIdempotency :execrows +INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id) +VALUES ($1, $2, $3, $4) +ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING; + + +-- name: GetIdempotentMessage :one +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at +FROM message_idempotency AS i +JOIN messages AS m ON m.id = i.message_id +WHERE i.user_id = $1 + AND i.chat_id = $2 + AND i.idempotency_key = $3; \ No newline at end of file diff --git a/db/queries/messages.sql b/db/queries/messages.sql new file mode 100644 index 00000000..f77fb037 --- /dev/null +++ b/db/queries/messages.sql @@ -0,0 +1,19 @@ +-- name: CreateMessage :one +INSERT INTO messages (chat_id, sender_id, text) +VALUES ($1, $2, $3) +RETURNING *; + +-- name: GetMessagesByChatID :many + +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = sqlc.arg(chat_id) + AND ( + sqlc.narg(after_created_at)::timestamptz IS NULL + OR (created_at, id) > + (sqlc.narg(after_created_at)::timestamptz, + sqlc.narg(after_id)::uuid + ) + ) +ORDER BY created_at ASC, id ASC +LIMIT sqlc.arg(limit_count); diff --git a/db/queries/messages_attachments.sql b/db/queries/messages_attachments.sql new file mode 100644 index 00000000..96c77bb8 --- /dev/null +++ b/db/queries/messages_attachments.sql @@ -0,0 +1,11 @@ +-- name: CreateMessagesAttachments :execrows +INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) +SELECT $1, $2, $3, $4 +FROM attachments +WHERE id = $2 AND chat_id = $3; + +-- name: GetMessagesAttachments :many +SELECT attachment_id, position +FROM messages_attachments +WHERE message_id = sqlc.arg(message_id) +ORDER BY position ASC; \ No newline at end of file diff --git a/db/queries/users.sql b/db/queries/users.sql new file mode 100644 index 00000000..730dfe25 --- /dev/null +++ b/db/queries/users.sql @@ -0,0 +1,21 @@ + +-- name: CreateUser :one + +INSERT INTO users (username, display_name, password_hash) +VALUES ($1, $2, $3) +RETURNING id, username, display_name, password_hash, created_at; + + + +-- name: GetUserByID :one + +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE id = $1; + + + +-- name: GetUserByUsername :one +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE username = $1; diff --git a/evidence/lab4-final/README.md b/evidence/lab4-final/README.md new file mode 100644 index 00000000..6481f2c2 --- /dev/null +++ b/evidence/lab4-final/README.md @@ -0,0 +1,26 @@ +# Проверки lab4 на отдельном Compose-стенде + +В HTTP-проверках адрес стенда был `http://127.0.0.1:18094`. PostgreSQL и Redis для интеграционных тестов доступны только на loopback. Запускать сценарии отказов на синтетических данных в отдельном проекте. + +Артефакты базовой реализации уровня 4, повторные проверки текущей реализации и команды запуска указаны в [отчёте](../../REPORT.md). [Манифест снимка](source.json) фиксирует исходники прогона. Полная проверка PostgreSQL, Redis и перезапусков повторена на [финальном снимке](../lab4-grade5/go.txt). + +## Профиль памяти + +Лёгкий сценарий использует публичный PNG 320 × 200. Для тяжёлого сценария можно воспроизвести PNG и JPEG размером 6000 × 6000 командой `generate` из [генератора](upload-fixture-generator.go.txt), сохранив его с расширением `.go` во временной папке и запустив `go run probe.go generate` из этой папки. Генерация выполняется отдельно от замера. + +```sh +python3 student-tests/lab4/upload_memory.py \ + --base-url http://127.0.0.1:18094 --project lab4-final-offline \ + --fixtures /path/to/generated --output /path/to/results +``` + +Двенадцать клиентов выходят из одного барьера. Два профиля запускаются последовательно. `memory.peak` считан до и после каждого профиля, поэтому он относится к жизни контейнера. В обоих профилях счётчик вырос. Сырые выборки `memory.current` приложены в JSON. Сборщик не записывает пароли, токены и тела запросов. + +## Отказы и границы + +- `attachment_pixel_limit.py --base-url URL --output DIR` создаёт валидный PNG 6500 × 6200 и ожидает `413`. +- `attachment_runtime_faults.py --base-url URL --output DIR --probe PATH --probe-image IMAGE` останавливает загрузившую реплику, проверяет SHA-256 с другой, восстанавливает API и затем отключает RustFS. Для прямого HTTP-запроса собери [probe](direct-probe.go.txt) под Linux и архитектуру Docker-хоста. `COMPOSE_PROJECT_NAME` должен начинаться с `lab4-final-`, а `COMPOSE_FILE` при необходимости включает override. +- `attachment_compensation.py --spec SPEC --output DIR --as-hook` принимает JSON с `project`, `runtime` (каталог локальных secrets), `snapshot` (корень копии) и `base` (массив аргументов, начиная с `["docker","compose","-p",project]`). Используется выделенный Compose-проект. Тест включает SIGKILL API и короткий grace. [Результат](../lab4-grade5/compensation.json) · [Наблюдения](../lab4-grade5/compensation-events.jsonl) +- `s3_infrastructure.py --project PROJECT --base-url URL --output DIR` проверяет bootstrap, настройки прав, анонимный доступ и консоль. + +В [первой попытке](faults-first-attempt.txt) утилита nc закрыла соединение до получения HTTP-ответа. Повтор из сетевого пространства API дал `live=200`, `ready=503`, upload `503`. [Результат](s3-failure.json) diff --git a/evidence/lab4-final/attachment-http.json b/evidence/lab4-final/attachment-http.json new file mode 100644 index 00000000..f6fea2fd --- /dev/null +++ b/evidence/lab4-final/attachment-http.json @@ -0,0 +1,50 @@ +{ + "attachment_acl_bytes_and_idempotency": { + "detail": "", + "elapsed_seconds": 0.928, + "requirement": "lab4_grade3", + "status": "PASS" + }, + "attachment_bucket_security": { + "detail": "configure hooks.attachment_bucket_security before starting the fault", + "elapsed_seconds": 0.0, + "requirement": "lab4_grade4", + "status": "PENDING_INFRA" + }, + "attachment_compensation": { + "detail": "configure hooks.attachment_compensation before starting the fault", + "elapsed_seconds": 0.0, + "requirement": "lab4_grade4", + "status": "PENDING_INFRA" + }, + "attachment_negative_uploads": { + "detail": "", + "elapsed_seconds": 0.776, + "requirement": "lab4_grade4", + "status": "PASS" + }, + "attachment_parallel_memory": { + "detail": "configure hooks.attachment_parallel_memory before starting the fault", + "elapsed_seconds": 0.0, + "requirement": "lab4_grade4", + "status": "PENDING_INFRA" + }, + "attachment_pixels_limit": { + "detail": "configure hooks.attachment_pixels_limit before starting the fault", + "elapsed_seconds": 0.0, + "requirement": "lab4_grade3", + "status": "PENDING_INFRA" + }, + "attachment_s3_failure": { + "detail": "configure hooks.attachment_s3_failure before starting the fault", + "elapsed_seconds": 0.0, + "requirement": "lab4_grade3", + "status": "PENDING_INFRA" + }, + "public_lb_endpoint": { + "detail": "", + "elapsed_seconds": 0.278, + "requirement": "inherited_contract_or_requested_check", + "status": "PASS" + } +} diff --git a/evidence/lab4-final/direct-probe.go.txt b/evidence/lab4-final/direct-probe.go.txt new file mode 100644 index 00000000..2cbcdc5c --- /dev/null +++ b/evidence/lab4-final/direct-probe.go.txt @@ -0,0 +1,15 @@ +package main +import ("bufio";"bytes";"encoding/json";"io";"net/http";"os";"strings";"time") +func main(){ + in:=bufio.NewReader(os.Stdin) + line,err:=in.ReadString('\n'); if err!=nil {panic(err)} + token,err:=in.ReadString('\n'); if err!=nil {panic(err)} + body,err:=io.ReadAll(in); if err!=nil {panic(err)} + fields:=strings.Fields(line) + req,err:=http.NewRequest(fields[0],"http://127.0.0.1:8080"+fields[1],bytes.NewReader(body)); if err!=nil {panic(err)} + if t:=strings.TrimSpace(token); t!="" {req.Header.Set("Authorization","Bearer "+t); req.Header.Set("Content-Type","image/png")} + res,err:=(&http.Client{Timeout:12*time.Second}).Do(req); if err!=nil {panic(err)} + defer res.Body.Close() + b,err:=io.ReadAll(res.Body); if err!=nil {panic(err)} + if err:=json.NewEncoder(os.Stdout).Encode(map[string]any{"status":res.StatusCode,"body":string(b),"instance":res.Header.Get("X-Instance-Id")});err!=nil{panic(err)} +} diff --git a/evidence/lab4-final/faults-first-attempt.txt b/evidence/lab4-final/faults-first-attempt.txt new file mode 100644 index 00000000..27954309 --- /dev/null +++ b/evidence/lab4-final/faults-first-attempt.txt @@ -0,0 +1,7 @@ +Traceback (most recent call last): + File "test-output/faults.py", line 77, in + check((live,rd,upload)==(200,503,503) and b'unavailable' in body, "wrong API outage statuses") + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "snapshot/tests/client.py", line 24, in check + raise Failure(message) +client.Failure: wrong API outage statuses diff --git a/evidence/lab4-final/file-failover.json b/evidence/lab4-final/file-failover.json new file mode 100644 index 00000000..565bc272 --- /dev/null +++ b/evidence/lab4-final/file-failover.json @@ -0,0 +1,7 @@ +{ + "source": "1c2fc2215977", + "replica": "38de6c779806", + "failures": 0, + "seconds": 0.023, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" +} diff --git a/evidence/lab4-final/memory-heavy.json b/evidence/lab4-final/memory-heavy.json new file mode 100644 index 00000000..18a51fb2 --- /dev/null +++ b/evidence/lab4-final/memory-heavy.json @@ -0,0 +1,506 @@ +{ + "clients": 12, + "files": [ + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + }, + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + }, + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + }, + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + }, + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + }, + { + "name": "load.png", + "mime": "image/png", + "bytes": 642711 + }, + { + "name": "load.jpg", + "mime": "image/jpeg", + "bytes": 1269896 + } + ], + "statuses": { + "201": 4, + "429": 8 + }, + "results": [ + { + "index": 0, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 1.4950219999955152 + }, + { + "index": 1, + "status": 201, + "instance": "38de6c779806", + "seconds": 1.4105649160046596 + }, + { + "index": 2, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.9557449580024695 + }, + { + "index": 3, + "status": 429, + "instance": "38de6c779806", + "seconds": 1.0306972920006956 + }, + { + "index": 4, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.9493740839970997 + }, + { + "index": 5, + "status": 429, + "instance": "38de6c779806", + "seconds": 1.0500210840036743 + }, + { + "index": 6, + "status": 429, + "instance": "1c2fc2215977", + "seconds": 1.0534772089959006 + }, + { + "index": 7, + "status": 429, + "instance": "1c2fc2215977", + "seconds": 1.0391598749993136 + }, + { + "index": 8, + "status": 429, + "instance": "38de6c779806", + "seconds": 1.0649052080043475 + }, + { + "index": 9, + "status": 429, + "instance": "1c2fc2215977", + "seconds": 1.0194469999987632 + }, + { + "index": 10, + "status": 429, + "instance": "38de6c779806", + "seconds": 1.0226270000057411 + }, + { + "index": 11, + "status": 429, + "instance": "1c2fc2215977", + "seconds": 1.034560625004815 + } + ], + "replicas": { + "38de6c779806": { + "before_peak": 150097920, + "after_peak": 294215680, + "limit": 536870912, + "samples": [ + [ + 0.075, + 149831680 + ], + [ + 0.134, + 148865024 + ], + [ + 0.184, + 148680704 + ], + [ + 0.24, + 148602880 + ], + [ + 0.293, + 148631552 + ], + [ + 0.347, + 148795392 + ], + [ + 0.401, + 148758528 + ], + [ + 0.455, + 149409792 + ], + [ + 0.505, + 154324992 + ], + [ + 0.559, + 162516992 + ], + [ + 0.606, + 177471488 + ], + [ + 0.659, + 188268544 + ], + [ + 0.714, + 194568192 + ], + [ + 0.764, + 202862592 + ], + [ + 0.816, + 213696512 + ], + [ + 0.869, + 219779072 + ], + [ + 0.921, + 228106240 + ], + [ + 0.974, + 238596096 + ], + [ + 1.026, + 255332352 + ], + [ + 1.077, + 268062720 + ], + [ + 1.128, + 274460672 + ], + [ + 1.179, + 282320896 + ], + [ + 1.231, + 292990976 + ], + [ + 1.285, + 292786176 + ], + [ + 1.337, + 292786176 + ], + [ + 1.388, + 293543936 + ], + [ + 1.442, + 293543936 + ], + [ + 1.492, + 293556224 + ], + [ + 1.56, + 293953536 + ], + [ + 1.606, + 293445632 + ], + [ + 1.67, + 293560320 + ], + [ + 1.716, + 293740544 + ], + [ + 1.767, + 293765120 + ], + [ + 1.818, + 292229120 + ], + [ + 1.871, + 292200448 + ], + [ + 1.923, + 292634624 + ], + [ + 1.976, + 292233216 + ], + [ + 2.037, + 291532800 + ], + [ + 2.094, + 291721216 + ], + [ + 2.14, + 291598336 + ], + [ + 2.191, + 291680256 + ] + ], + "events_delta": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "restart_delta": 0, + "same_start": true, + "oom_killed": false + }, + "1c2fc2215977": { + "before_peak": 147574784, + "after_peak": 430215168, + "limit": 536870912, + "samples": [ + [ + 0.066, + 146915328 + ], + [ + 0.117, + 146403328 + ], + [ + 0.171, + 146513920 + ], + [ + 0.223, + 146685952 + ], + [ + 0.274, + 146599936 + ], + [ + 0.328, + 146632704 + ], + [ + 0.386, + 146812928 + ], + [ + 0.437, + 147116032 + ], + [ + 0.493, + 161148928 + ], + [ + 0.543, + 169295872 + ], + [ + 0.595, + 183906304 + ], + [ + 0.649, + 190574592 + ], + [ + 0.702, + 196771840 + ], + [ + 0.758, + 205017088 + ], + [ + 0.81, + 215502848 + ], + [ + 0.861, + 223907840 + ], + [ + 0.914, + 230154240 + ], + [ + 0.966, + 238407680 + ], + [ + 1.017, + 253476864 + ], + [ + 1.07, + 270123008 + ], + [ + 1.121, + 274477056 + ], + [ + 1.173, + 282595328 + ], + [ + 1.227, + 293322752 + ], + [ + 1.283, + 293048320 + ], + [ + 1.332, + 293031936 + ], + [ + 1.387, + 295157760 + ], + [ + 1.439, + 304148480 + ], + [ + 1.49, + 325140480 + ], + [ + 1.56, + 337682432 + ], + [ + 1.644, + 356155392 + ], + [ + 1.693, + 366571520 + ], + [ + 1.744, + 385736704 + ], + [ + 1.795, + 400392192 + ], + [ + 1.846, + 415043584 + ], + [ + 1.898, + 429740032 + ], + [ + 1.951, + 428744704 + ], + [ + 2.006, + 428699648 + ], + [ + 2.06, + 428515328 + ], + [ + 2.111, + 428564480 + ], + [ + 2.162, + 428535808 + ] + ], + "events_delta": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "restart_delta": 0, + "same_start": true, + "oom_killed": false + } + }, + "sampling_seconds": 0.05 +} diff --git a/evidence/lab4-final/memory-normal.json b/evidence/lab4-final/memory-normal.json new file mode 100644 index 00000000..224395b9 --- /dev/null +++ b/evidence/lab4-final/memory-normal.json @@ -0,0 +1,309 @@ +{ + "clients": 12, + "files": [ + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + }, + { + "name": "sample.png", + "mime": "image/png", + "bytes": 111817 + } + ], + "statuses": { + "201": 12 + }, + "results": [ + { + "index": 0, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.11950787499517901 + }, + { + "index": 1, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.1198934999993071 + }, + { + "index": 2, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.11025404199608602 + }, + { + "index": 3, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.1579494590041577 + }, + { + "index": 4, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.08275312499608845 + }, + { + "index": 5, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.0824889580035233 + }, + { + "index": 6, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.09312024999962887 + }, + { + "index": 7, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.15798666600312572 + }, + { + "index": 8, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.16619729199737776 + }, + { + "index": 9, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.09303424999961862 + }, + { + "index": 10, + "status": 201, + "instance": "1c2fc2215977", + "seconds": 0.134490292002738 + }, + { + "index": 11, + "status": 201, + "instance": "38de6c779806", + "seconds": 0.1474745419982355 + } + ], + "replicas": { + "38de6c779806": { + "before_peak": 78581760, + "after_peak": 82251776, + "limit": 536870912, + "samples": [ + [ + 0.074, + 77082624 + ], + [ + 0.126, + 76701696 + ], + [ + 0.177, + 76554240 + ], + [ + 0.228, + 76816384 + ], + [ + 0.281, + 76812288 + ], + [ + 0.338, + 77144064 + ], + [ + 0.396, + 76996608 + ], + [ + 0.45, + 77750272 + ], + [ + 0.499, + 78159872 + ], + [ + 0.552, + 78475264 + ], + [ + 0.601, + 80609280 + ], + [ + 0.652, + 80601088 + ], + [ + 0.706, + 80605184 + ], + [ + 0.759, + 80637952 + ], + [ + 0.812, + 80785408 + ], + [ + 0.864, + 80646144 + ] + ], + "events_delta": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "restart_delta": 0, + "same_start": true, + "oom_killed": false + }, + "1c2fc2215977": { + "before_peak": 78098432, + "after_peak": 80687104, + "limit": 536870912, + "samples": [ + [ + 0.074, + 76730368 + ], + [ + 0.126, + 76226560 + ], + [ + 0.178, + 76251136 + ], + [ + 0.229, + 76247040 + ], + [ + 0.281, + 76398592 + ], + [ + 0.338, + 76300288 + ], + [ + 0.396, + 76271616 + ], + [ + 0.45, + 76718080 + ], + [ + 0.499, + 77037568 + ], + [ + 0.552, + 78348288 + ], + [ + 0.602, + 78499840 + ], + [ + 0.654, + 78487552 + ], + [ + 0.707, + 78499840 + ], + [ + 0.759, + 78700544 + ], + [ + 0.812, + 78569472 + ], + [ + 0.864, + 78299136 + ] + ], + "events_delta": { + "low": 0, + "high": 0, + "max": 0, + "oom": 0, + "oom_kill": 0, + "oom_group_kill": 0 + }, + "restart_delta": 0, + "same_start": true, + "oom_killed": false + } + }, + "sampling_seconds": 0.05 +} diff --git a/evidence/lab4-final/pixel-limit.json b/evidence/lab4-final/pixel-limit.json new file mode 100644 index 00000000..52e7d9de --- /dev/null +++ b/evidence/lab4-final/pixel-limit.json @@ -0,0 +1,6 @@ +{ + "width": 6500, + "height": 6200, + "encoded_bytes": 39249, + "upload_status": 413 +} diff --git a/evidence/lab4-final/restart-after.txt b/evidence/lab4-final/restart-after.txt new file mode 100644 index 00000000..f06ead15 --- /dev/null +++ b/evidence/lab4-final/restart-after.txt @@ -0,0 +1,9 @@ +api 1c2fc22159776084669ac7ffdfc7ab18778485feb6810b4c534fb90baec30d62 2026-11-14T20:59:38.277558213Z +api 38de6c779806333887d344e72f5de2b4a4336b6ef02cc8bfe400809ad3b711d3 2026-11-14T20:59:38.462321713Z +attachment-cleanup e573c9b2f3fac6dbd03b8a941e8cff44199cbe5f8dcf0662d847ca60dfef3f33 2026-11-14T20:59:44.4007593Z +lb 85e0635f043975812d11820b400edbf1e14428928e526e5a644e41fea9e210ce 2026-11-14T20:59:44.279379091Z +migrate 8b1303443156719ba3c74bd9771f889a47689cdc447a254eb88ac44cb6ca6e3c 2026-11-14T20:58:31.100201127Z +postgres aabd87a6deea77a10e45a268726a2aa5bdd3111d1b62e8a5238c4efe96fa9a9d 2026-11-14T20:59:31.482838877Z lab4-final-offline_postgres_data +redis a6b9d8046bc219a2ee1f9624aab5a898f129f9081efcd241c8962941d65cb1b4 2026-11-14T20:59:31.973372085Z lab4-final-offline_redis_data +s3 1dd57160fa6a64fa3ea2e95e097c623d2beaef1926fc8d5f75e2aedff601eaa0 2026-11-14T20:59:32.286071627Z lab4-final-offline_rustfs_data +s3-init 7a365a86ababc0976e6b9cb4192bfa223a38e5baec615cc0a03bf369ec7a9042 2026-11-14T20:58:31.102788085Z diff --git a/evidence/lab4-final/restart-before.txt b/evidence/lab4-final/restart-before.txt new file mode 100644 index 00000000..2d6a76d9 --- /dev/null +++ b/evidence/lab4-final/restart-before.txt @@ -0,0 +1,9 @@ +api 1c2fc22159776084669ac7ffdfc7ab18778485feb6810b4c534fb90baec30d62 2026-11-14T20:58:31.887295044Z +api 38de6c779806333887d344e72f5de2b4a4336b6ef02cc8bfe400809ad3b711d3 2026-11-14T20:57:52.42195897Z +attachment-cleanup e573c9b2f3fac6dbd03b8a941e8cff44199cbe5f8dcf0662d847ca60dfef3f33 2026-11-14T20:58:00.616614543Z +lb 85e0635f043975812d11820b400edbf1e14428928e526e5a644e41fea9e210ce 2026-11-14T20:57:58.431995375Z +migrate 8b1303443156719ba3c74bd9771f889a47689cdc447a254eb88ac44cb6ca6e3c 2026-11-14T20:58:31.100201127Z +postgres aabd87a6deea77a10e45a268726a2aa5bdd3111d1b62e8a5238c4efe96fa9a9d 2026-11-14T20:57:46.098278675Z lab4-final-offline_postgres_data +redis a6b9d8046bc219a2ee1f9624aab5a898f129f9081efcd241c8962941d65cb1b4 2026-11-14T20:57:45.953826092Z lab4-final-offline_redis_data +s3 1dd57160fa6a64fa3ea2e95e097c623d2beaef1926fc8d5f75e2aedff601eaa0 2026-11-14T20:58:49.826562677Z lab4-final-offline_rustfs_data +s3-init 7a365a86ababc0976e6b9cb4192bfa223a38e5baec615cc0a03bf369ec7a9042 2026-11-14T20:58:31.102788085Z diff --git a/evidence/lab4-final/s3-failure.json b/evidence/lab4-final/s3-failure.json new file mode 100644 index 00000000..3dac2113 --- /dev/null +++ b/evidence/lab4-final/s3-failure.json @@ -0,0 +1,8 @@ +{ + "fault": "s3_unreachable", + "live_status": 200, + "ready_status": 503, + "upload_status": 503, + "false_ready_attachments": 0, + "restored_original_checksum_equal": true +} diff --git a/evidence/lab4-final/s3-security.json b/evidence/lab4-final/s3-security.json new file mode 100644 index 00000000..bfa824e0 --- /dev/null +++ b/evidence/lab4-final/s3-security.json @@ -0,0 +1,68 @@ +{ + "s3_api_bucket_scope_and_iam_denials": { + "detail": "", + "elapsed_seconds": 5.314, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_api_delete_effect": { + "detail": "", + "elapsed_seconds": 0.928, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_api_effective_policy": { + "detail": "", + "elapsed_seconds": 0.945, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_bootstrap_preserves_object_and_volume": { + "detail": "", + "elapsed_seconds": 2.742, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_console_closed_on_shared_network": { + "detail": "", + "elapsed_seconds": 0.749, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_fixture_cleanup": { + "detail": "", + "elapsed_seconds": 4.046, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_restart_preserves_original_bytes": { + "detail": "", + "elapsed_seconds": 6.263, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_setup_and_scenarios": { + "detail": "", + "elapsed_seconds": 26.37, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_two_replicas_after_restart": { + "detail": "", + "elapsed_seconds": 0.474, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_two_replicas_before": { + "detail": "", + "elapsed_seconds": 0.667, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_unsigned_known_object_403": { + "detail": "", + "elapsed_seconds": 0.301, + "requirement": "requested_operational_check", + "status": "PASS" + } +} diff --git a/evidence/lab4-final/s3-security.jsonl b/evidence/lab4-final/s3-security.jsonl new file mode 100644 index 00000000..54994145 --- /dev/null +++ b/evidence/lab4-final/s3-security.jsonl @@ -0,0 +1,44 @@ +{"bindings": [{"HostIp": "127.0.0.1", "HostPort": "18094"}], "event": "public_lb_endpoint", "port": 18094, "time_utc": "2026-11-14T20:58:41Z", "time_utc_ms": "2026-11-14T20:58:41.908+00:00"} +{"api_ids": ["1c2fc22159776084669ac7ffdfc7ab18778485feb6810b4c534fb90baec30d62", "38de6c779806333887d344e72f5de2b4a4336b6ef02cc8bfe400809ad3b711d3"], "event": "s3_api_topology", "lb_id": "85e0635f043975812d11820b400edbf1e14428928e526e5a644e41fea9e210ce", "lb_image": "haproxy:3.2.24-alpine", "observed_instances": ["1c2fc2215977", "38de6c779806"], "same_token": true, "time_utc": "2026-11-14T20:58:42Z", "time_utc_ms": "2026-11-14T20:58:42.146+00:00"} +{"detail": "", "elapsed_seconds": 0.667, "event": "case", "name": "s3_two_replicas_before", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:58:42Z", "time_utc_ms": "2026-11-14T20:58:42.147+00:00"} +{"control_bucket": "student-control-37a866cfecd04359ac1e1988712566b0", "create_probe_bucket": "student-forbidden-37a866cfecd04359ac1e1988712566b0", "event": "s3_initial", "fixture_prefix": "student-tests/s3-37a866cfecd04359ac1e1988712566b0/", "id": "1dd57160fa6a64fa3ea2e95e097c623d2beaef1926fc8d5f75e2aedff601eaa0", "image": "rustfs/rustfs:1.0.4", "payload_sha256": "15beb9311f947cccd73f69f39c44114ac8f7144c3b73b158610eba32f19fbcc8", "payload_size_bytes": 65824, "project": "lab4-final-offline", "running": true, "started_at": "2026-11-14T20:58:40.118706673Z", "time_utc": "2026-11-14T20:58:45Z", "time_utc_ms": "2026-11-14T20:58:45.585+00:00", "volumes": ["lab4-final-offline_rustfs_data"]} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "api_put_allowed_object", "s3_error_code": null, "time_utc": "2026-11-14T20:58:46Z", "time_utc_ms": "2026-11-14T20:58:46.220+00:00"} +{"byte_equal": true, "event": "s3_roundtrip", "phase": "before_bootstrap", "sha256": "15beb9311f947cccd73f69f39c44114ac8f7144c3b73b158610eba32f19fbcc8", "size_bytes": 65824, "time_utc": "2026-11-14T20:58:46Z", "time_utc_ms": "2026-11-14T20:58:46.751+00:00"} +{"attempt": 1, "event": "s3_bootstrap", "exit_code": 0, "time_utc": "2026-11-14T20:58:47Z", "time_utc_ms": "2026-11-14T20:58:47.491+00:00"} +{"byte_equal": true, "event": "s3_roundtrip", "phase": "bootstrap_1", "sha256": "15beb9311f947cccd73f69f39c44114ac8f7144c3b73b158610eba32f19fbcc8", "size_bytes": 65824, "time_utc": "2026-11-14T20:58:48Z", "time_utc_ms": "2026-11-14T20:58:48.065+00:00"} +{"attempt": 2, "event": "s3_bootstrap", "exit_code": 0, "time_utc": "2026-11-14T20:58:48Z", "time_utc_ms": "2026-11-14T20:58:48.830+00:00"} +{"byte_equal": true, "event": "s3_roundtrip", "phase": "bootstrap_2", "sha256": "15beb9311f947cccd73f69f39c44114ac8f7144c3b73b158610eba32f19fbcc8", "size_bytes": 65824, "time_utc": "2026-11-14T20:58:49Z", "time_utc_ms": "2026-11-14T20:58:49.494+00:00"} +{"detail": "", "elapsed_seconds": 2.742, "event": "case", "name": "s3_bootstrap_preserves_object_and_volume", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:58:49Z", "time_utc_ms": "2026-11-14T20:58:49.494+00:00"} +{"after": {"id": "1dd57160fa6a64fa3ea2e95e097c623d2beaef1926fc8d5f75e2aedff601eaa0", "image": "rustfs/rustfs:1.0.4", "running": true, "started_at": "2026-11-14T20:58:49.826562677Z", "volumes": ["lab4-final-offline_rustfs_data"]}, "before": {"id": "1dd57160fa6a64fa3ea2e95e097c623d2beaef1926fc8d5f75e2aedff601eaa0", "image": "rustfs/rustfs:1.0.4", "running": true, "started_at": "2026-11-14T20:58:40.118706673Z", "volumes": ["lab4-final-offline_rustfs_data"]}, "event": "s3_restart", "ready_seconds": 5.481, "time_utc": "2026-11-14T20:58:55Z", "time_utc_ms": "2026-11-14T20:58:55.101+00:00"} +{"byte_equal": true, "event": "s3_roundtrip", "phase": "after_restart", "sha256": "15beb9311f947cccd73f69f39c44114ac8f7144c3b73b158610eba32f19fbcc8", "size_bytes": 65824, "time_utc": "2026-11-14T20:58:55Z", "time_utc_ms": "2026-11-14T20:58:55.755+00:00"} +{"detail": "", "elapsed_seconds": 6.263, "event": "case", "name": "s3_restart_preserves_original_bytes", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:58:55Z", "time_utc_ms": "2026-11-14T20:58:55.757+00:00"} +{"bucket_list_status": 403, "event": "s3_unsigned", "object_get_status": 403, "time_utc": "2026-11-14T20:58:56Z", "time_utc_ms": "2026-11-14T20:58:56.059+00:00"} +{"detail": "", "elapsed_seconds": 0.301, "event": "case", "name": "s3_unsigned_known_object_403", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:58:56Z", "time_utc_ms": "2026-11-14T20:58:56.059+00:00"} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "api_list_allowed_bucket", "s3_error_code": null, "time_utc": "2026-11-14T20:58:56Z", "time_utc_ms": "2026-11-14T20:58:56.543+00:00"} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "prepare_control_bucket", "s3_error_code": null, "time_utc": "2026-11-14T20:58:57Z", "time_utc_ms": "2026-11-14T20:58:57.099+00:00"} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "prepare_control_object", "s3_error_code": null, "time_utc": "2026-11-14T20:58:57Z", "time_utc_ms": "2026-11-14T20:58:57.721+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 4, "operation": "api_other_bucket_put", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:58:58Z", "time_utc_ms": "2026-11-14T20:58:58.255+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 3, "operation": "api_other_bucket_get", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:58:58Z", "time_utc_ms": "2026-11-14T20:58:58.825+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 4, "operation": "api_other_bucket_delete", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:58:59Z", "time_utc_ms": "2026-11-14T20:58:59.375+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 3, "operation": "api_other_bucket_list", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:58:59Z", "time_utc_ms": "2026-11-14T20:58:59.847+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 4, "operation": "api_create_bucket", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:59:00Z", "time_utc_ms": "2026-11-14T20:59:00.330+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 1, "operation": "api_iam_user_list", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:59:00Z", "time_utc_ms": "2026-11-14T20:59:00.867+00:00"} +{"authorization_denied": true, "event": "s3_operation", "exit_code": 1, "operation": "api_iam_policy_list", "s3_error_code": "AccessDenied", "time_utc": "2026-11-14T20:59:01Z", "time_utc_ms": "2026-11-14T20:59:01.373+00:00"} +{"detail": "", "elapsed_seconds": 5.314, "event": "case", "name": "s3_api_bucket_scope_and_iam_denials", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:01Z", "time_utc_ms": "2026-11-14T20:59:01.373+00:00"} +{"event": "s3_effective_policy", "grants": [{"action": "s3:DeleteObject", "resource": "arn:aws:s3:::messenger-attachments/*"}, {"action": "s3:GetBucketLocation", "resource": "arn:aws:s3:::messenger-attachments"}, {"action": "s3:GetObject", "resource": "arn:aws:s3:::messenger-attachments/*"}, {"action": "s3:ListBucket", "resource": "arn:aws:s3:::messenger-attachments"}, {"action": "s3:PutObject", "resource": "arn:aws:s3:::messenger-attachments/*"}], "iam_admin_grants": 0, "inherited_groups": 0, "policy_name": "messenger-attachments-api-v1", "root_credentials_distinct": true, "time_utc": "2026-11-14T20:59:02Z", "time_utc_ms": "2026-11-14T20:59:02.317+00:00"} +{"detail": "", "elapsed_seconds": 0.945, "event": "case", "name": "s3_api_effective_policy", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:02Z", "time_utc_ms": "2026-11-14T20:59:02.318+00:00"} +{"console_curl_exit": 7, "console_http_status": "000", "control_status": 200, "event": "s3_console", "time_utc": "2026-11-14T20:59:03Z", "time_utc_ms": "2026-11-14T20:59:03.066+00:00"} +{"detail": "", "elapsed_seconds": 0.749, "event": "case", "name": "s3_console_closed_on_shared_network", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:03Z", "time_utc_ms": "2026-11-14T20:59:03.067+00:00"} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "api_delete_allowed_object", "s3_error_code": null, "time_utc": "2026-11-14T20:59:03Z", "time_utc_ms": "2026-11-14T20:59:03.561+00:00"} +{"authorization_denied": false, "event": "s3_operation", "exit_code": 0, "operation": "api_list_after_delete", "s3_error_code": null, "time_utc": "2026-11-14T20:59:03Z", "time_utc_ms": "2026-11-14T20:59:03.995+00:00"} +{"detail": "", "elapsed_seconds": 0.928, "event": "case", "name": "s3_api_delete_effect", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:03Z", "time_utc_ms": "2026-11-14T20:59:03.996+00:00"} +{"bindings": [{"HostIp": "127.0.0.1", "HostPort": "18094"}], "event": "public_lb_endpoint", "port": 18094, "time_utc": "2026-11-14T20:59:04Z", "time_utc_ms": "2026-11-14T20:59:04.404+00:00"} +{"api_ids": ["1c2fc22159776084669ac7ffdfc7ab18778485feb6810b4c534fb90baec30d62", "38de6c779806333887d344e72f5de2b4a4336b6ef02cc8bfe400809ad3b711d3"], "event": "s3_api_topology", "lb_id": "85e0635f043975812d11820b400edbf1e14428928e526e5a644e41fea9e210ce", "lb_image": "haproxy:3.2.24-alpine", "observed_instances": ["1c2fc2215977", "38de6c779806"], "same_token": true, "time_utc": "2026-11-14T20:59:04Z", "time_utc_ms": "2026-11-14T20:59:04.470+00:00"} +{"detail": "", "elapsed_seconds": 0.474, "event": "case", "name": "s3_two_replicas_after_restart", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:04Z", "time_utc_ms": "2026-11-14T20:59:04.471+00:00"} +{"already_absent": false, "event": "s3_cleanup", "exit_code": 0, "operation": "object remove", "target": "root/messenger-attachments/student-tests/s3-37a866cfecd04359ac1e1988712566b0/roundtrip-37a866cfecd04359ac1e1988712566b0.bin", "time_utc": "2026-11-14T20:59:05Z", "time_utc_ms": "2026-11-14T20:59:05.546+00:00"} +{"already_absent": false, "event": "s3_cleanup", "exit_code": 0, "operation": "object remove", "target": "root/student-control-37a866cfecd04359ac1e1988712566b0/sentinel.bin", "time_utc": "2026-11-14T20:59:06Z", "time_utc_ms": "2026-11-14T20:59:06.143+00:00"} +{"already_absent": false, "event": "s3_cleanup", "exit_code": 0, "operation": "object remove", "target": "root/student-control-37a866cfecd04359ac1e1988712566b0/api-write.bin", "time_utc": "2026-11-14T20:59:06Z", "time_utc_ms": "2026-11-14T20:59:06.827+00:00"} +{"already_absent": false, "event": "s3_cleanup", "exit_code": 0, "operation": "bucket remove", "target": "root/student-control-37a866cfecd04359ac1e1988712566b0", "time_utc": "2026-11-14T20:59:07Z", "time_utc_ms": "2026-11-14T20:59:07.803+00:00"} +{"already_absent": true, "event": "s3_cleanup", "exit_code": 5, "operation": "bucket remove", "target": "root/student-forbidden-37a866cfecd04359ac1e1988712566b0", "time_utc": "2026-11-14T20:59:08Z", "time_utc_ms": "2026-11-14T20:59:08.516+00:00"} +{"detail": "", "elapsed_seconds": 4.046, "event": "case", "name": "s3_fixture_cleanup", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:08Z", "time_utc_ms": "2026-11-14T20:59:08.517+00:00"} +{"detail": "", "elapsed_seconds": 26.37, "event": "case", "name": "s3_setup_and_scenarios", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-14T20:59:08Z", "time_utc_ms": "2026-11-14T20:59:08.517+00:00"} diff --git a/evidence/lab4-final/source.json b/evidence/lab4-final/source.json new file mode 100644 index 00000000..87305acb --- /dev/null +++ b/evidence/lab4-final/source.json @@ -0,0 +1,106 @@ +{ + "files": { + "Dockerfile": "17e5c9fa261dbe5395791213f605787824957da6567586ce990e44e3fd5a1236", + "Dockerfile.migrate": "0ab04cba4c2bb033c4d7c7d3a5984d8a27e8f355ff90e6023c3b7912d2211dec", + "cmd/api/main.go": "e9568a3a1098b01fded10cdc502dac652bf6a3a5761914c6860c9f45b56ed269", + "cmd/api/poolstats.go": "75c0ca950b85eaee3c0802e3001c61ba6436874b3be12cd38a5feb907b91528e", + "cmd/attachment-cleanup/main.go": "347895c38d76e0c2d866dc4e1eb5b7555b447b796a77c5da8c2d6e7198f614d3", + "cmd/migrate/main.go": "6e9d413a86f53cf5f2e83d5ed0308af9b2c33e69fcb3d061b19bf480e352f257", + "cmd/migrate/main_test.go": "a88f2252d70caf644904331182dc3afd4df48b825a0329a82482798ec90b5d46", + "cmd/migrate/postgres_test.go": "37401cd60657af560d52e727b08861f444ab18355d9decd9ceeeddb009d079ca", + "cmd/migrate/roundtrip_test.go": "25b3425447b1dbafec83db2e5a7fc8172031908d5f5d57ce7b0d36a97850f8fa", + "compose.yaml": "e67836432e988a1035a91ed028471e801fcf0827fb6ff4070f1e583abc06d787", + "db/migrations/00001_init_schema.sql": "32223afbc24f8250de055690c9e86299ebf2cd7a0d3ef2c59d3b2a5aaa41e84e", + "db/migrations/00002_add_created_at_defaults.sql": "e0680838294b93f916d727a3a20d83aca7c3f74077202c9696b1fc73cda14649", + "db/migrations/00003_drop_session_table.sql": "1b666cee95b9eb1210b0a303de95994bd3c24628e9d9fdb1e1f3318edb756c3d", + "db/migrations/00004_message_idempotency.sql": "e7b3dd5a056a702dfde7acba6666430477db0c7e03ae03fb270e145d6303b9b2", + "db/migrations/00005_attachments.sql": "c28ca64fb84d6c60dc181e4cb0022461894274b85970df84bc38bb9984585bb4", + "db/migrations/00006_allow_attachment_only_messages.sql": "ad56ecd0e8b88e0665cdc144aaa5d934ea7f48047cd702ce6f8832b6cf0bded3", + "db/migrations/embed.go": "a42e52ab05bc89b557a5bc6b3f0215883d99c4a039a8b32d6c19e66b2f769508", + "db/queries/attachments.sql": "15beeee95faf5b608179b0977a7dd45cd14a1c14df022e4c313194cb307646a0", + "db/queries/chat_members.sql": "e70940e35f325550041c290a571901ead4a6f93dece857ffa229f2f1c83eddf8", + "db/queries/chats.sql": "8c52c762da78eb3a3eb7402b17ec87bd7fc2e1387f6ddb8e297b4f2d75beca7b", + "db/queries/message_idempotence.sql": "34259e4beee9cc07de4dec95c1d588222c7d756d02ac6564b54d6dbec36678db", + "db/queries/messages.sql": "022c9f7aade55345f6cebd31f38084e73cb9a1419d56fa5316eaa36c9da52860", + "db/queries/messages_attachments.sql": "753e5b46dbe94b1043fbe19c83850e697feef7ea9ccc348496b71cb6f9906ddb", + "db/queries/users.sql": "0d73a6d46f4d4a1b0af37b750a208c07a38249f76974b41748f32c1949f990df", + "go.mod": "0dbf79ae49966e763868e20b453addaecbb74085dd7b0579271f0839ca8a17cd", + "go.sum": "30264a0cbc88f40ac455abb621624a54aa956bedd390c07deef4f67812efc982", + "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", + "infra/haproxy.cfg": "895a178e7354de61a5d40a2fe18b7b7c6d1d3d5faf0e5a5faa72783e7b983cb8", + "infra/s3/bootstrap.sh": "baa4b0b6fa4bd2a76b194b672ef2ca8575858ef6abc7ddfbede3eec3534bc8ce", + "internal/config/config.go": "142318dac05e3070358e7642c4b09813fe6bdb4d1b80e1154954d5e4e5f3fa59", + "internal/database/postgres.go": "3925c0bc9ae01bff9a37138a0dd19b38074bc2a70536b3a148f568b952516a21", + "internal/database/redis.go": "5b85a60bb0d1cb156c61bcd3c518c41c955f53c1d12da265b5e3a5dd7a397602", + "internal/database/sqlc/attachments.sql.go": "15c2c687f98031d4f360ca9db10670f9dca3d7188d3a5b7df9d7705d55632871", + "internal/database/sqlc/chat_members.sql.go": "9e8cd177f6f888c72d19857fd4748fbb2020770c1af772a463fe3d1b813b37ee", + "internal/database/sqlc/chats.sql.go": "3dccf4b0ffa918ec5a3b66026b3dab4027456b499486359881000d184944802c", + "internal/database/sqlc/db.go": "3ca71a7f007a39987cae838da64303419dd804d2d1b6227676f47177971d76cf", + "internal/database/sqlc/message_idempotence.sql.go": "ebd31e6f369483eed8d6978dab1687552eed742a612b5ee4090499fe8a48a73f", + "internal/database/sqlc/messages.sql.go": "224d6d8bd21426759ca6500e0eb3563564d51e639f1b6c64db4b3a43a5241c2a", + "internal/database/sqlc/messages_attachments.sql.go": "821d863a01079edae53805769821f1dc5eb3e32620ce61efd7326bb0253d2204", + "internal/database/sqlc/models.go": "df06356e705031dfc6226efd736f432853e026c26c30424675b99179819d422e", + "internal/database/sqlc/users.sql.go": "b954abd5b5964df39c20d857a60105513af84c8df68ce1ac9421da541a7800f4", + "internal/model/attachment.go": "877de511f5996c263c63c3e406ed67ee6609833f4946074594e1d438f8dce53a", + "internal/model/chat.go": "e0213f8360ea106b3627aa50cc87710153ff8e353475b42c2becdb9a32592660", + "internal/model/message.go": "4cebc4963e2eb104e85a1df6e8a93d6745c9ae42425df2d04bf6319e1d17b12d", + "internal/model/session.go": "b455d730e4e3efc95c6d1fe74ab43d86153e4bf863c09efd36bfe57efc669bc2", + "internal/model/user.go": "4bfd2d78e72f1c80b12541f2abc8500613e8f4d101575bd5d74fac22b52e3617", + "internal/repository/attachment.go": "75069a930484f0c464a3d237c42aedaa5cc58b606068b8834bc78184426f7d46", + "internal/repository/attachment_tx.go": "28fa001ae601fcc9b54e1e097b6e895d2f6ffb40b006c48a8862e0c822a2d896", + "internal/repository/chat.go": "7ef8dca39ff371f16d351b073199c25f719db22c7a525ab76ea7aca948efa157", + "internal/repository/errors.go": "9e3f77340ff87e41edc87380f795759701980685478737e7e4e1f9737e2634fa", + "internal/repository/message.go": "2f92e2808317850ca5873cc26f8e6c4d83104e8fc957a89304fc74ba2f64cbf2", + "internal/repository/postgres/attachment.go": "c37254509a9bf24158b73b8ae805c229a28b2aa2a203f7106d6c8a1ac69b10cd", + "internal/repository/postgres/chat.go": "d757c579e5b6599ab9366aef46fd59f9652247f3d947615aa46c919746b65ea5", + "internal/repository/postgres/errors.go": "580e5e599f7af8a403404c6850986e88efceda6393d21e772ac108be34004ba9", + "internal/repository/postgres/message.go": "a2057f6fe6ba2a93a7f98bdbd799baf043999811e9fa7bb649e4f0d60138b6c2", + "internal/repository/postgres/user.go": "eb6770e5cdd96b16a6e9d2116af89a6dd466dfbca4f4c179fe06c30c487e53c4", + "internal/repository/postgres/utils.go": "6c5c83a8ad8ecf611812fbf8c90735e2590a09dec88cd914a10f6b36a9f041b7", + "internal/repository/redis/errors.go": "ab56571d1d0b7d13b7d7ff868916f7e506c5a54c29e6bb211b3021ad4bf601ab", + "internal/repository/redis/session.go": "c1654fea2ba47734a5403d6f6cbb2fe70fbe7e6e37166ae75f338d3490981de1", + "internal/repository/session.go": "071285ff4d8033a6512a7256583a1cb32de782d6fd9f1696fac109927f7db699", + "internal/repository/user.go": "bee65136137d97bea2240411c96a93cd945b1b9145452416f7a83733a1fd28e7", + "internal/security/password.go": "8f74bef9216a2d12aa734e04fa786eac6b31f1378fe42f729fe1b97495c10487", + "internal/service/attachment.go": "4b901d85d93d6dffde1bf76dd532b4b6845afd55ea7a8f9aaea983eaead1e9da", + "internal/service/attachment_cleanup.go": "8e0112bacba6d90447c2be9884f34e5ca0be808af172e4cdaa4e6dea0f906dbb", + "internal/service/auth.go": "71b8960640866e515e9060159af750c418d97a6373d2616c3b59b764172aa54c", + "internal/service/chat.go": "bc7c8bea0066d8bbd926f78d2c100f0bd9a48c29fb542c5cfa3f2ae415299ef5", + "internal/service/errors.go": "702b41496201ec42a69591eaa7c5154bf2aaf52aecbc1318a87d4a32ccc5d5df", + "internal/service/helper.go": "390320fa296d9637630e6ac0d90c1229002b578a29186cad414896d5f4d1b8a6", + "internal/service/message.go": "9556a18a5679d7e44b5cdcf329488185d2b90cadb3180af926df4bb6ab3f9d0d", + "internal/service/user.go": "542051ef3eeff15d340979a615a7e0da3d21ed7ed93bea2547ab52d8e303b5e2", + "internal/storage/errors.go": "9cc85df576c98b29c9cc9ed244bdae2aa3d03361c3bdb0307c2940202e99c377", + "internal/storage/object_cleanup_storage.go": "ef50236baefe3e3aea8ca64264ae3c2e5fc89c245b9b1d7cfd207d354f0be2c9", + "internal/storage/object_writer.go": "15ca1257b9c42f64a42f19a4bbb487e9d2950aed2fdf87959fe6c2d763998b64", + "internal/storage/s3/client.go": "c6939f4dfd9ca8a7380010ff8551480cbff200b621af6f8b9ce444bd2a3f8a32", + "internal/storage/s3/errors.go": "f5e058597d8d3cba2e5db7e81eb4c0a87f7d8b1c1d17a0aff2409801be0cd701", + "internal/transport/http/handler/attachment.go": "885b2c4d24cbf0f5bfa9def4f6b5b4a7621919b47c3906d89a297abcf0c73da8", + "internal/transport/http/handler/attachment_types.go": "90210075401601bacb25ae70dea2e19a203f537cf0f5aff1369679619bb87811", + "internal/transport/http/handler/auth.go": "f175a570d47ca38ce627d0bc88b6f6a2adbf648345229fa961ea0144717744f1", + "internal/transport/http/handler/auth_types.go": "5ffcd182606a77d538efb5f1a51df23b8a324a605b5d226026048d34c409ebdb", + "internal/transport/http/handler/chat.go": "e5214ebf94ac9b8f37537e1a3fc844a2e8ae4e9b5aa159fe2b6e4da9ab2342f0", + "internal/transport/http/handler/chat_types.go": "d181bce0f003749d813c1254df4c8ebcc1b99ff506b61efa6899da4a499e95f4", + "internal/transport/http/handler/health.go": "1abf04652ea0b69bd87321b95e1a921d3c5ad3c04207232c2a9408bf722e550a", + "internal/transport/http/handler/health_types.go": "02ee3031e10fbb279e020f409fbd905ae2640aa306b2c558c3f8e6dcfe9efa80", + "internal/transport/http/handler/message.go": "540065ffc9090848fc25d560f1ad9734d4f0d45cdf0b5907a0f196f9efa8ede3", + "internal/transport/http/handler/message_types.go": "270a9504a3001b1dc02a1f62cdaed12d69af2471340674e50022c609643dcb34", + "internal/transport/http/handler/user.go": "86fd65ed176e9a83dc27c1112ff8384bc5aa32a92410290fb505d715ca167775", + "internal/transport/http/handler/user_types.go": "0c4450be649afe5edf4e8d3ac9baada79deb9e1423ace56b87aa227ebd0eb998", + "internal/transport/http/httputils/helper.go": "59c092950b69f5390427d0c6c1f90d2c33630d87a42bef363aed704960cb3a22", + "internal/transport/http/middleware/auth.go": "413d7fb5f8573a808c2eb4e36becd29b4d7edffe5b096b26fc07ac22a0285430", + "internal/transport/http/middleware/context.go": "f6db0e807659f30b05cae91f73c70d87f359760c57110e0d25b49a65bd0394a5", + "internal/transport/http/middleware/logging.go": "a477fda512953eb9f46266537c12ace9cc992ba8f036a5b041f9f8295bfd2097", + "internal/transport/http/router.go": "2841bac8988517591539336ba20718b4da7bae31ab81bf56e6b2b9bb59b0a297", + "scripts/restart.sh": "5f70beb4337aed16881a0728229f02dce0e95d42028e0088acd52aa0a76eb019", + "sqlc.yaml": "4e72de8f0b44d6d2b6a44da89f86dd5326e85077a5e7a8ae71816a5dff1fbaa3" + }, + "source_tree_sha256": "f1dcdad773a3fb8de430ab9f85825dc61eba2954255dc5e0400c6c30ab9f84da", + "submission_whitespace_only": { + "db/migrations/00001_init_schema.sql": "48db014273f78be0f6eb97fc521cccc54f851dc99e3688d480d3a6ff7b9b6bbe", + "db/migrations/00005_attachments.sql": "033d8884cecb50b5067453a97f2d7c51e51331a1911d9ddf8687437afa59443c", + "db/queries/chat_members.sql": "c80b6e50c321ff94317545ea2f9fdee1dfd69f564d8638f7b111c8ec091ef03a", + "db/queries/messages.sql": "f6f6382f27d698b1c2af2305d45699226ef10806b938b3dca78ec9087a622b7b", + "internal/database/sqlc/messages.sql.go": "9836433d9645e6dd3bd66273ce15a53691f79093016c416fb24ffd085e62831e" + } +} diff --git a/evidence/lab4-final/upload-fixture-generator.go.txt b/evidence/lab4-final/upload-fixture-generator.go.txt new file mode 100644 index 00000000..5c53ed27 --- /dev/null +++ b/evidence/lab4-final/upload-fixture-generator.go.txt @@ -0,0 +1,323 @@ +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "image" + "image/jpeg" + "image/png" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "time" +) + +type settings struct { + Address, Instance, Token, Chat, CancelChat, EmptyCancelChat string +} +type status struct { + HTTP int `json:"http_status"` + Instance string `json:"instance"` + Code string `json:"error_code,omitempty"` + Retry string `json:"retry_after,omitempty"` + SHA string `json:"sha256,omitempty"` + Size int `json:"size_bytes,omitempty"` + RequestID string `json:"request_id,omitempty"` + Elapsed float64 `json:"elapsed_seconds"` +} +type event map[string]any + +var events []event + +func record(name string, data any) { + events = append(events, event{"event": name, "time_utc": time.Now().UTC().Format(time.RFC3339Nano), "data": data}) +} +func check(ok bool, description string) error { + if !ok { + return errors.New(description) + } + return nil +} +func decode(resp *http.Response, start time.Time) (status, error) { + defer resp.Body.Close() + var value struct { + Error struct { + Code string `json:"code"` + } `json:"error"` + SHA string `json:"sha256"` + Size int `json:"size_bytes"` + } + err := json.NewDecoder(resp.Body).Decode(&value) + return status{HTTP: resp.StatusCode, Instance: resp.Header.Get("X-Instance-Id"), Retry: resp.Header.Get("Retry-After"), + Code: value.Error.Code, SHA: value.SHA, Size: value.Size, RequestID: resp.Header.Get("X-Request-Id"), Elapsed: time.Since(start).Seconds()}, err +} +func path(chat, filename string) string { + return "/api/v1/chats/" + chat + "/attachments?filename=" + filename +} +func post(c settings, filename, mime string, body []byte) (status, error) { + start := time.Now() + req, err := http.NewRequest("POST", "http://"+c.Address+path(c.Chat, filename), bytes.NewReader(body)) + if err != nil { + return status{}, err + } + req.Header.Set("Content-Type", mime) + req.Header.Set("Authorization", "Bearer "+c.Token) + // One connection per upload makes the simultaneous requests independent. + req.Close = true + resp, err := (&http.Client{Timeout: 12 * time.Second}).Do(req) + if err != nil { + return status{Elapsed: time.Since(start).Seconds()}, err + } + return decode(resp, start) +} + +type pending struct { + conn net.Conn + reader *bufio.Reader + start time.Time +} + +func begin(c settings, chat, filename string, length int, expect bool) (*pending, error) { + conn, err := net.DialTimeout("tcp", c.Address, 2*time.Second) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(8 * time.Second)) + expectation := "" + if expect { + expectation = "Expect: 100-continue\r\n" + } + _, err = fmt.Fprintf(conn, "POST %s HTTP/1.1\r\nHost: %s\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n%sConnection: close\r\n\r\n", path(chat, filename), c.Address, c.Token, length, expectation) + if err != nil { + conn.Close() + return nil, err + } + return &pending{conn, bufio.NewReader(conn), time.Now()}, nil +} +func (p *pending) provisional() error { + response, err := http.ReadResponse(p.reader, nil) + if err != nil { + return err + } + response.Body.Close() + return check(response.StatusCode == 100, fmt.Sprintf("expected provisional 100, got %d", response.StatusCode)) +} +func (p *pending) finish(body []byte) (status, error) { + if _, err := p.conn.Write(body); err != nil { + return status{}, err + } + response, err := http.ReadResponse(p.reader, nil) + if err != nil { + return status{}, err + } + return decode(response, p.start) +} +func valid(value status, expectedInstance string, payload []byte) error { + return check(value.HTTP == 201 && (expectedInstance == "" || value.Instance == expectedInstance) && + value.SHA == fmt.Sprintf("%x", sha256.Sum256(payload)) && value.Size == len(payload) && + value.RequestID != "", fmt.Sprintf("expected valid 201 from %s, got %d from %s", expectedInstance, value.HTTP, value.Instance)) +} +func controlled(c settings) error { + body, err := os.ReadFile("/input/sample.png") + if err != nil { + return err + } + first, err := begin(c, c.Chat, "held-a.png", len(body), true) + if err != nil { + return err + } + defer first.conn.Close() + if err := first.provisional(); err != nil { + return err + } + second, err := begin(c, c.Chat, "held-b.png", len(body), true) + if err != nil { + return err + } + defer second.conn.Close() + if err := second.provisional(); err != nil { + return err + } + record("both_slots_held", event{"provisional_statuses": []int{100, 100}, "body_bytes_sent": 0}) + limited, err := post(c, "limited.png", "image/png", body) + record("third_upload", limited) + if err != nil { + return err + } + if err := check(limited.HTTP == 429 && limited.Code == "rate_limited" && limited.Retry == "1" && + limited.Instance == c.Instance && limited.RequestID != "" && limited.Elapsed >= .9 && limited.Elapsed <= 1.5, + "third upload did not return the expected 429, Retry-After and one-second wait"); err != nil { + return err + } + for _, cancel := range []struct { + chat string + size int + }{ + {c.CancelChat, len(body)}, {c.EmptyCancelChat, 0}, + } { + waiter, err := begin(c, cancel.chat, "cancel.png", cancel.size, false) + if err != nil { + return err + } + time.Sleep(220 * time.Millisecond) + waiter.conn.Close() + record("waiter_disconnected", event{"chat_id": cancel.chat, "declared_bytes": cancel.size, "body_bytes_sent": 0, + "disconnect_after_seconds": time.Since(waiter.start).Seconds()}) + } + // Both slots stay occupied for longer than the wait timeout, so server logs + // distinguish cancellation from a later slot grant. + time.Sleep(1150 * time.Millisecond) + r, err := first.finish(body) + record("first_holder_finished", r) + if err != nil { + return err + } + if err := valid(r, c.Instance, body); err != nil { + return err + } + r, err = post(c, "after-release.png", "image/png", body) + record("upload_after_release", r) + if err != nil { + return err + } + if err := valid(r, c.Instance, body); err != nil { + return err + } + r, err = second.finish(body) + record("second_holder_finished", r) + if err != nil { + return err + } + return valid(r, c.Instance, body) +} +func load(c settings) error { + pngData, err := os.ReadFile("/input/load.png") + if err != nil { + return err + } + jpegData, err := os.ReadFile("/input/load.jpg") + if err != nil { + return err + } + payloads := []struct { + name, mime string + data []byte + }{{"load.png", "image/png", pngData}, {"load.jpg", "image/jpeg", jpegData}} + const count = 12 + var wg, ready sync.WaitGroup + var active, peak atomic.Int32 + start := make(chan struct{}) + done := make(chan event, count) + for i := 0; i < count; i++ { + wg.Add(1) + ready.Add(1) + go func(i int) { + defer wg.Done() + p := payloads[i%len(payloads)] + ready.Done() + <-start + n := active.Add(1) + for current := peak.Load(); n > current && !peak.CompareAndSwap(current, n); current = peak.Load() { + } + value, err := post(c, p.name, p.mime, p.data) + active.Add(-1) + detail := event{"request": i + 1, "file": p.name, "response": value} + if err == nil { + if value.HTTP == 201 { + err = valid(value, "", p.data) + } else if value.HTTP == 429 { + err = check(value.Code == "rate_limited" && value.Retry == "1" && value.Instance != "" && value.RequestID != "", "invalid rate limit response") + } else { + err = fmt.Errorf("unexpected status %d", value.HTTP) + } + } + if err != nil { + detail["failure"] = err.Error() + } + done <- detail + }(i) + } + ready.Wait() + close(start) + wg.Wait() + close(done) + failures := 0 + for detail := range done { + record("concurrent_response", detail) + if detail["failure"] != nil { + failures++ + } + } + record("concurrent_peak", event{"workers": count, "client_in_flight_peak": peak.Load()}) + return check(peak.Load() >= 10 && failures == 0, "load had fewer than 10 overlapping requests or contained unexpected responses") +} +func generate(directory string) error { + const width, height = 6000, 6000 + img := image.NewNRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + i := y*img.Stride + x*4 + img.Pix[i] = byte(x/12 + y/36) + img.Pix[i+1] = byte(x/24 - y/9) + img.Pix[i+2] = byte((x / 8) ^ (y / 16)) + img.Pix[i+3] = 255 + } + } + for _, name := range []string{"load.png", "load.jpg"} { + f, err := os.Create(filepath.Join(directory, name)) + if err != nil { + return err + } + if strings.HasSuffix(name, "png") { + err = png.Encode(f, img) + } else { + err = jpeg.Encode(f, img, &jpeg.Options{Quality: 90}) + } + f.Close() + if err != nil { + return err + } + } + return nil +} +func main() { + if len(os.Args) != 2 { + os.Exit(2) + } + if os.Args[1] == "generate" { + if err := generate("."); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + return + } + var c settings + if err := json.NewDecoder(os.Stdin).Decode(&c); err != nil { + os.Exit(2) + } + var err error + switch os.Args[1] { + case "controlled": + err = controlled(c) + case "load": + err = load(c) + default: + err = errors.New("unknown mode") + } + report := event{"events": events} + if err != nil { + report["failure"] = err.Error() + } + json.NewEncoder(os.Stdout).Encode(report) + if err != nil { + os.Exit(1) + } +} diff --git a/evidence/lab4-grade5/README.md b/evidence/lab4-grade5/README.md new file mode 100644 index 00000000..e3b41b25 --- /dev/null +++ b/evidence/lab4-grade5/README.md @@ -0,0 +1,30 @@ +# Проверка двусторонней сверки + +Понадобится новая копия проекта, Docker с Compose, Python 3.9+ и OpenSSL. +Перед экспериментом убедитесь, что выбранные порты свободны. + +```sh +export LAB4_HTTP_PORT=18096 LAB4_POSTGRES_PORT=15496 +export LAB4_CLEANUP_DRY_RUN=true +sh scripts/init-local.sh +docker compose -p lab4-final-grade5 --env-file .env up -d --build --wait +python3 student-tests/lab4/reconcile_experiment.py \ + --stack "$PWD" --project lab4-final-grade5 \ + --base-url http://127.0.0.1:18096 --output /tmp/lab4-reconcile +docker compose -p lab4-final-grade5 --env-file .env down --volumes +``` + +Эксперимент требует отдельный свежий стенд: он удаляет строки и файлы +синтетических участников, затем заполняет тестовые данные для замера. Рабочий +grace составляет 10 минут. Для воспроизводимости тест сокращает его до +4 секунд в сценариях ошибок и 1 секунды в замерах без ошибок. + +[Журнал эксперимента](experiment.txt) · [Счётчики и время всех проходов](reconcile.json) +· [Сборка и запуск](compose-build.txt) · [Миграции 1–6](migrate.txt) + +Повторная проверка на Go 1.27.1 / Docker Compose 2.39.4: +[make check](check.txt) · [smoke](smoke.txt) · [перезапуск](persistence.txt) +· [failover](failover.txt) · [Go с PostgreSQL и Redis](go.txt). +Полный Go-прогон выполнен на чистой копии со всеми настройками PostgreSQL и Redis. Для Go-тестов дополнительный Compose override публиковал Redis только на loopback. Хуки перезапуска обращались к выделенному Redis и этому же проекту. + +Повтор аварийной уборки на чистой версии: [результаты](compensation.json) · [события](compensation-events.jsonl). diff --git a/evidence/lab4-grade5/check.txt b/evidence/lab4-grade5/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/lab4-grade5/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/lab4-grade5/compensation-events.jsonl b/evidence/lab4-grade5/compensation-events.jsonl new file mode 100644 index 00000000..9a1c1152 --- /dev/null +++ b/evidence/lab4-grade5/compensation-events.jsonl @@ -0,0 +1,26 @@ +{"event": "sql_isolation", "time_utc": "2026-11-15T08:49:07Z", "time_utc_ms": "2026-11-15T08:49:07.400+00:00", "value": "read committed"} +{"detail": "", "elapsed_seconds": 1.369, "event": "case", "name": "setup", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:07Z", "time_utc_ms": "2026-11-15T08:49:07.556+00:00"} +{"dry_run": true, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:18Z", "time_utc_ms": "2026-11-15T08:49:18.502+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:20Z", "time_utc_ms": "2026-11-15T08:49:20.726+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:22Z", "time_utc_ms": "2026-11-15T08:49:22.489+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:23Z", "time_utc_ms": "2026-11-15T08:49:23.325+00:00"} +{"applied": true, "dry_run_kept": true, "event": "retention", "fresh_preserved": true, "held_keys_preserved": true, "keys": {"commit": "attachments/cleanup-2020d04dc6-busy-commit", "fresh": "attachments/cleanup-2020d04dc6-fresh", "linked": "attachments/2b6a237d-3e88-44fe-bd1b-d50f36be2e29", "orphan": "attachments/cleanup-2020d04dc6-orphan", "outside": "other/cleanup-2020d04dc6", "rowless": "attachments/cleanup-2020d04dc6-busy-rollback", "unlinked": "attachments/c7c9db8e-3ccd-4499-985d-ffd1858f5a58"}, "last_modified": {"attachments/2b6a237d-3e88-44fe-bd1b-d50f36be2e29": "2026-11-15T08:49:07.589000+00:00", "attachments/c7c9db8e-3ccd-4499-985d-ffd1858f5a58": "2026-11-15T08:49:07.567000+00:00", "attachments/cleanup-2020d04dc6-busy-commit": "2026-11-15T08:49:08.408000+00:00", "attachments/cleanup-2020d04dc6-busy-rollback": "2026-11-15T08:49:08.555000+00:00", "attachments/cleanup-2020d04dc6-orphan": "2026-11-15T08:49:08.081000+00:00", "other/cleanup-2020d04dc6": "2026-11-15T08:49:08.227000+00:00"}, "referenced_after_lock": true, "referenced_preserved": true, "repeated": true, "rowless_after_lock_deleted": true, "time_utc": "2026-11-15T08:49:23Z", "time_utc_ms": "2026-11-15T08:49:23.557+00:00"} +{"detail": "", "elapsed_seconds": 16.001, "event": "case", "name": "retention", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:23Z", "time_utc_ms": "2026-11-15T08:49:23.557+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:24Z", "time_utc_ms": "2026-11-15T08:49:24.957+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:33Z", "time_utc_ms": "2026-11-15T08:49:33.961+00:00"} +{"deleted_after_grace": true, "event": "after_s3_before_db", "http_status": 500, "key": "attachments/0623c85b-59c3-4915-9ce9-b9deb8ca257a", "last_modified": "2026-11-15T08:49:23.855000+00:00", "rows": 0, "survived_young": true, "time_utc": "2026-11-15T08:49:34Z", "time_utc_ms": "2026-11-15T08:49:34.174+00:00"} +{"detail": "", "elapsed_seconds": 10.617, "event": "case", "name": "fault_after_put", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:34Z", "time_utc_ms": "2026-11-15T08:49:34.175+00:00"} +{"bytes_declared": 111817, "bytes_sent": 256, "event": "client_disconnect", "objects_created": 0, "rows": 0, "subsequent_status": 201, "time_utc": "2026-11-15T08:49:35Z", "time_utc_ms": "2026-11-15T08:49:35.279+00:00"} +{"detail": "", "elapsed_seconds": 1.104, "event": "case", "name": "aborted_body", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:35Z", "time_utc_ms": "2026-11-15T08:49:35.279+00:00"} +{"api": "f9bf1d922ba338cfa81fce5a3e4dae7707e6003329d91fe8fd2fa16afed68fa6", "api_ip": "172.19.0.5", "backend_pid": 46, "event": "kill_boundary", "insert_waiting": true, "key": "attachments/7675f167-66f7-44aa-bb3f-d82345b21919", "row_count": 0, "time_utc": "2026-11-15T08:49:36Z", "time_utc_ms": "2026-11-15T08:49:36.588+00:00"} +{"api": "f9bf1d922ba338cfa81fce5a3e4dae7707e6003329d91fe8fd2fa16afed68fa6", "backend_wait_before_unblock": "advisory", "client_connection_check_interval": "0", "event": "sigkill", "exit_code": 137, "signal": 9, "time_utc": "2026-11-15T08:49:37Z", "time_utc_ms": "2026-11-15T08:49:37.153+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:38Z", "time_utc_ms": "2026-11-15T08:49:38.156+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:46Z", "time_utc_ms": "2026-11-15T08:49:46.142+00:00"} +{"api": "f9bf1d922ba338cfa81fce5a3e4dae7707e6003329d91fe8fd2fa16afed68fa6", "backend_released": true, "client": {"error": "Failure"}, "event": "killed_api", "key": "attachments/7675f167-66f7-44aa-bb3f-d82345b21919", "old_deleted": true, "rows": 0, "time_utc": "2026-11-15T08:49:46Z", "time_utc_ms": "2026-11-15T08:49:46.321+00:00", "young_preserved": true} +{"detail": "", "elapsed_seconds": 11.341, "event": "case", "name": "killed_api_after_put", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:46Z", "time_utc_ms": "2026-11-15T08:49:46.622+00:00"} +{"dry_run": false, "event": "one_shot", "exit_code": 0, "grace_seconds": 8, "time_utc": "2026-11-15T08:49:56Z", "time_utc_ms": "2026-11-15T08:49:56.506+00:00"} +{"downstream_bytes": 0, "event": "lost_response", "key": "attachments/c52011a2-c360-458e-a806-c1d37a1a9ffd", "rows": 1, "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", "survived_apply": true, "time_utc": "2026-11-15T08:49:56Z", "time_utc_ms": "2026-11-15T08:49:56.539+00:00", "upstream_status": 201} +{"detail": "", "elapsed_seconds": 9.917, "event": "case", "name": "lost_response", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:49:56Z", "time_utc_ms": "2026-11-15T08:49:56.539+00:00"} +{"deleted_seconds": 8.679, "event": "periodic", "key": "attachments/cleanup-2020d04dc6-periodic", "time_utc": "2026-11-15T08:50:05Z", "time_utc_ms": "2026-11-15T08:50:05.441+00:00"} +{"during": "EXISTS", "event": "sigterm", "exit_code": 0, "key": "attachments/cleanup-2020d04dc6-signal", "lock_released": true, "restart_deleted": true, "time_utc": "2026-11-15T08:50:16Z", "time_utc_ms": "2026-11-15T08:50:16.194+00:00"} +{"detail": "", "elapsed_seconds": 19.777, "event": "case", "name": "periodic_sigterm", "requirement": "requested_operational_check", "status": "PASS", "time_utc": "2026-11-15T08:50:16Z", "time_utc_ms": "2026-11-15T08:50:16.319+00:00"} diff --git a/evidence/lab4-grade5/compensation.json b/evidence/lab4-grade5/compensation.json new file mode 100644 index 00000000..9c85aa65 --- /dev/null +++ b/evidence/lab4-grade5/compensation.json @@ -0,0 +1,44 @@ +{ + "aborted_body": { + "detail": "", + "elapsed_seconds": 1.104, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "fault_after_put": { + "detail": "", + "elapsed_seconds": 10.617, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "killed_api_after_put": { + "detail": "", + "elapsed_seconds": 11.341, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "lost_response": { + "detail": "", + "elapsed_seconds": 9.917, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "periodic_sigterm": { + "detail": "", + "elapsed_seconds": 19.777, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "retention": { + "detail": "", + "elapsed_seconds": 16.001, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "setup": { + "detail": "", + "elapsed_seconds": 1.369, + "requirement": "requested_operational_check", + "status": "PASS" + } +} diff --git a/evidence/lab4-grade5/compose-build.txt b/evidence/lab4-grade5/compose-build.txt new file mode 100644 index 00000000..f1a437f3 --- /dev/null +++ b/evidence/lab4-grade5/compose-build.txt @@ -0,0 +1,236 @@ +#1 [internal] load local bake definitions +#1 reading from stdin 1.51kB done +#1 DONE 0.0s + +#2 [migrate internal] load build definition from Dockerfile.migrate +#2 transferring dockerfile: 448B done +#2 DONE 0.0s + +#3 [attachment-cleanup internal] load build definition from Dockerfile +#3 transferring dockerfile: 855B done +#3 DONE 0.0s + +#4 [migrate internal] load metadata for docker.io/library/golang:1.27.1-alpine +#4 DONE 2.6s + +#5 [attachment-cleanup internal] load metadata for docker.io/library/alpine:3.22 +#5 DONE 2.6s + +#6 [api internal] load .dockerignore +#6 transferring context: 196B done +#6 DONE 0.0s + +#7 [migrate builder 1/9] FROM docker.io/library/golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 +#7 resolve docker.io/library/golang:1.27.1-alpine@sha256:8a5910f31396cd4d89662f56c68b3ae31d374308270a1c3bd96672ee5ed43414 0.0s done +#7 DONE 0.0s + +#8 [migrate runtime 1/3] FROM docker.io/library/alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f +#8 resolve docker.io/library/alpine:3.22@sha256:49340b346798875b5592e5d420241f2f9cfba5f47e62ebb1446a1b34734e898f 0.0s done +#8 DONE 0.0s + +#9 [migrate internal] load build context +#9 transferring context: 41.50kB done +#9 DONE 0.0s + +#10 [migrate builder 4/7] RUN go mod download +#10 CACHED + +#11 [migrate builder 3/7] COPY go.mod go.sum ./ +#11 CACHED + +#12 [migrate builder 5/7] COPY cmd/migrate/ ./cmd/migrate/ +#12 CACHED + +#13 [attachment-cleanup internal] load build context +#13 transferring context: 138.51kB 0.0s done +#13 DONE 0.0s + +#14 [api builder 2/9] WORKDIR /app +#14 CACHED + +#15 [api builder 3/9] COPY go.mod go.sum ./ +#15 CACHED + +#16 [api builder 4/9] RUN go mod download +#16 CACHED + +#17 [api builder 5/9] COPY cmd/api/ ./cmd/api/ +#17 CACHED + +#18 [migrate builder 6/7] COPY db/migrations/ ./db/migrations/ +#18 DONE 0.0s + +#19 [api builder 6/9] COPY cmd/attachment-cleanup/ ./cmd/attachment-cleanup/ +#19 DONE 0.0s + +#20 [api builder 7/9] COPY internal/ ./internal/ +#20 DONE 0.0s + +#21 [api builder 8/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +#21 ... + +#22 [migrate builder 7/7] RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /app/migrate ./cmd/migrate +#22 DONE 19.9s + +#21 [api builder 8/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +#21 ... + +#23 [migrate runtime 2/3] RUN addgroup -S app && adduser -S app -G app +#23 CACHED + +#24 [migrate runtime 3/3] WORKDIR /app +#24 CACHED + +#23 [attachment-cleanup runtime 2/3] RUN addgroup -S app && adduser -S app -G app +#23 CACHED + +#25 [migrate stage-1 4/4] COPY --from=builder /app/migrate ./migrate +#25 DONE 0.3s + +#21 [api builder 8/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +#21 ... + +#26 [migrate] exporting to image +#26 exporting layers +#26 exporting layers 1.1s done +#26 exporting manifest sha256:f4b196e5c4d235b5593dfdf9fcc9f7f3943bdf78a380d9cbf703449ec1fd730a 0.0s done +#26 exporting config sha256:4194b73a1ba046593f5e0d4a8dc79c9f0a1ac5ced1c2ff9f29fbf6d765ebebe3 0.1s done +#26 exporting attestation manifest sha256:5c34cb23b4efd9767c126fb43a0a87b63a436ee2299436d38a65183fba7f47e0 0.0s done +#26 exporting manifest list sha256:f78bc60117392a76f0f177361994171d58d74cf218a30eae6cb6a998de7654bf +#26 exporting manifest list sha256:f78bc60117392a76f0f177361994171d58d74cf218a30eae6cb6a998de7654bf done +#26 naming to docker.io/library/lab4-final-grade5-migrate:latest done +#26 unpacking to docker.io/library/lab4-final-grade5-migrate:latest 0.1s done +#26 DONE 1.4s + +#21 [api builder 8/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +#21 ... + +#27 [migrate] resolving provenance for metadata file +#27 DONE 0.1s + +#21 [api builder 8/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/server ./cmd/api +#21 DONE 39.9s + +#28 [attachment-cleanup builder 9/9] RUN CGO_ENABLED=0 GOOS=linux go build -o /app/attachment-cleanup ./cmd/attachment-cleanup +#28 DONE 1.5s + +#29 [attachment-cleanup cleanup 1/1] COPY --from=builder /app/attachment-cleanup ./attachment-cleanup +#29 DONE 0.0s + +#30 [api api 1/1] COPY --from=builder /app/server ./server +#30 DONE 0.1s + +#31 [attachment-cleanup] exporting to image +#31 exporting layers +#31 exporting layers 0.6s done +#31 exporting manifest sha256:2ac7bb862c98bfad9f7c17ef0df831bcaff7de5aa8448fa674e823fdc4e45606 +#31 exporting manifest sha256:2ac7bb862c98bfad9f7c17ef0df831bcaff7de5aa8448fa674e823fdc4e45606 done +#31 exporting config sha256:1d3c8c064add7ba3996ddaf974bbf81ae95af4cea677816405d7002dbd2c1577 done +#31 exporting attestation manifest sha256:a578b63ee12ddd77fef902967a1c135df5e179486c652f31ca589124d398cfd7 0.0s done +#31 exporting manifest list sha256:720d441943e7f9a8c89dd2e1e275c1a5c0e748973b2e00f0d5f80dae0f375883 0.0s done +#31 naming to docker.io/library/lab4-final-grade5-attachment-cleanup:latest done +#31 unpacking to docker.io/library/lab4-final-grade5-attachment-cleanup:latest 0.1s done +#31 DONE 0.7s + +#32 [attachment-cleanup] resolving provenance for metadata file +#32 DONE 0.0s + +#33 [api] exporting to image +#33 exporting layers 0.6s done +#33 exporting manifest sha256:09bff2b9862296c85b67d43968c0ac6b704bcdae387f9d57a90b60373ba7789b done +#33 exporting config sha256:fbd04f42a1ccb2f2cd4e4c51686ac0bf3ef02cd4d37a252cb0236858a476f511 done +#33 exporting attestation manifest sha256:2185a5dd97b2281771aafe47e1c3ad0a097eb6d1c14f2782e0c8021d1fb17a65 0.0s done +#33 exporting manifest list sha256:4cb2ecb3a51c6ad66a581c9169662d17eb5c6f9fd183ebcefdd87d3b7caf51c8 0.0s done +#33 naming to docker.io/library/lab4-final-grade5-api:latest done +#33 unpacking to docker.io/library/lab4-final-grade5-api:latest 0.1s done +#33 DONE 0.8s + +#34 [api] resolving provenance for metadata file +#34 DONE 0.0s + lab4-final-grade5-migrate Built + lab4-final-grade5-api Built + lab4-final-grade5-attachment-cleanup Built + Network lab4-final-grade5_default Creating + Network lab4-final-grade5_default Created + Volume lab4-final-grade5_postgres_data Creating + Volume lab4-final-grade5_postgres_data Created + Volume lab4-final-grade5_redis_data Creating + Volume lab4-final-grade5_redis_data Created + Volume lab4-final-grade5_rustfs_data Creating + Volume lab4-final-grade5_rustfs_data Created + Container lab4-final-grade5-redis-1 Creating + Container lab4-final-grade5-s3-1 Creating + Container lab4-final-grade5-postgres-1 Creating + Container lab4-final-grade5-s3-1 Created + Container lab4-final-grade5-s3-init-1 Creating + Container lab4-final-grade5-redis-1 Created + Container lab4-final-grade5-postgres-1 Created + Container lab4-final-grade5-migrate-1 Creating + Container lab4-final-grade5-s3-init-1 Created + Container lab4-final-grade5-migrate-1 Created + Container lab4-final-grade5-attachment-cleanup-1 Creating + Container lab4-final-grade5-api-2 Creating + Container lab4-final-grade5-api-1 Creating + Container lab4-final-grade5-api-1 Created + Container lab4-final-grade5-api-2 Created + Container lab4-final-grade5-lb-1 Creating + Container lab4-final-grade5-attachment-cleanup-1 Created + Container lab4-final-grade5-lb-1 Created + Container lab4-final-grade5-s3-1 Starting + Container lab4-final-grade5-postgres-1 Starting + Container lab4-final-grade5-redis-1 Starting + Container lab4-final-grade5-postgres-1 Started + Container lab4-final-grade5-postgres-1 Waiting + Container lab4-final-grade5-redis-1 Started + Container lab4-final-grade5-s3-1 Started + Container lab4-final-grade5-s3-1 Waiting + Container lab4-final-grade5-postgres-1 Healthy + Container lab4-final-grade5-migrate-1 Starting + Container lab4-final-grade5-s3-1 Healthy + Container lab4-final-grade5-s3-init-1 Starting + Container lab4-final-grade5-migrate-1 Started + Container lab4-final-grade5-s3-init-1 Started + Container lab4-final-grade5-redis-1 Waiting + Container lab4-final-grade5-postgres-1 Waiting + Container lab4-final-grade5-migrate-1 Waiting + Container lab4-final-grade5-s3-init-1 Waiting + Container lab4-final-grade5-migrate-1 Waiting + Container lab4-final-grade5-s3-init-1 Waiting + Container lab4-final-grade5-postgres-1 Waiting + Container lab4-final-grade5-redis-1 Healthy + Container lab4-final-grade5-s3-init-1 Exited + Container lab4-final-grade5-migrate-1 Exited + Container lab4-final-grade5-postgres-1 Healthy + Container lab4-final-grade5-s3-init-1 Exited + Container lab4-final-grade5-migrate-1 Exited + Container lab4-final-grade5-attachment-cleanup-1 Starting + Container lab4-final-grade5-postgres-1 Healthy + Container lab4-final-grade5-api-2 Starting + Container lab4-final-grade5-api-2 Started + Container lab4-final-grade5-api-1 Starting + Container lab4-final-grade5-attachment-cleanup-1 Started + Container lab4-final-grade5-api-1 Started + Container lab4-final-grade5-api-2 Waiting + Container lab4-final-grade5-api-1 Waiting + Container lab4-final-grade5-api-2 Healthy + Container lab4-final-grade5-api-1 Healthy + Container lab4-final-grade5-lb-1 Starting + Container lab4-final-grade5-lb-1 Started + Container lab4-final-grade5-redis-1 Waiting + Container lab4-final-grade5-api-2 Waiting + Container lab4-final-grade5-api-1 Waiting + Container lab4-final-grade5-lb-1 Waiting + Container lab4-final-grade5-attachment-cleanup-1 Waiting + Container lab4-final-grade5-s3-init-1 Waiting + Container lab4-final-grade5-s3-1 Waiting + Container lab4-final-grade5-postgres-1 Waiting + Container lab4-final-grade5-migrate-1 Waiting + Container lab4-final-grade5-attachment-cleanup-1 Healthy + Container lab4-final-grade5-redis-1 Healthy + Container lab4-final-grade5-lb-1 Healthy + Container lab4-final-grade5-migrate-1 Exited + Container lab4-final-grade5-postgres-1 Healthy + Container lab4-final-grade5-s3-init-1 Exited + Container lab4-final-grade5-s3-1 Healthy + Container lab4-final-grade5-api-2 Healthy + Container lab4-final-grade5-api-1 Healthy diff --git a/evidence/lab4-grade5/experiment.txt b/evidence/lab4-grade5/experiment.txt new file mode 100644 index 00000000..350c165e --- /dev/null +++ b/evidence/lab4-grade5/experiment.txt @@ -0,0 +1 @@ +PASS: bidirectional reconciliation, preservation, repeated apply, and cost profiles diff --git a/evidence/lab4-grade5/failover.txt b/evidence/lab4-grade5/failover.txt new file mode 100644 index 00000000..f5f8629f --- /dev/null +++ b/evidence/lab4-grade5/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18096" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 1, "first_recovery_seconds": 2.788} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-grade5/go.txt b/evidence/lab4-grade5/go.txt new file mode 100644 index 00000000..68ae11ff --- /dev/null +++ b/evidence/lab4-grade5/go.txt @@ -0,0 +1,507 @@ +? highload/cmd/api [no test files] +? highload/cmd/attachment-cleanup [no test files] +=== RUN TestConnectionSecretErrorsAreSafe +=== RUN TestConnectionSecretErrorsAreSafe/missing +=== RUN TestConnectionSecretErrorsAreSafe/empty +=== RUN TestConnectionSecretErrorsAreSafe/malformed +--- PASS: TestConnectionSecretErrorsAreSafe (0.01s) + --- PASS: TestConnectionSecretErrorsAreSafe/missing (0.00s) + --- PASS: TestConnectionSecretErrorsAreSafe/empty (0.00s) + --- PASS: TestConnectionSecretErrorsAreSafe/malformed (0.00s) +=== RUN TestEmbeddedMigrationsAreVersionedAndCollectable +--- PASS: TestEmbeddedMigrationsAreVersionedAndCollectable (0.00s) +=== RUN TestUnlockHasBudgetEvenWithDetachedContext +--- PASS: TestUnlockHasBudgetEvenWithDetachedContext (0.00s) +=== RUN TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL +--- PASS: TestPostgresMigrationsRepeatWithoutChangingDataOrLoggingSQL (0.14s) +=== RUN TestPostgresConcurrentMigratorsApplyEachVersionOnce +--- PASS: TestPostgresConcurrentMigratorsApplyEachVersionOnce (1.06s) +=== RUN TestPostgresMigrationFailureRollsBackAndPreservesCause +--- PASS: TestPostgresMigrationFailureRollsBackAndPreservesCause (0.04s) +=== RUN TestPostgresMigrationThreeDownTwoDownOneAndReapply +--- PASS: TestPostgresMigrationThreeDownTwoDownOneAndReapply (0.12s) +PASS +ok highload/cmd/migrate 2.263s +? highload/db/migrations [no test files] +? highload/internal/config [no test files] +? highload/internal/database [no test files] +? highload/internal/database/sqlc [no test files] +? highload/internal/model [no test files] +? highload/internal/repository [no test files] +? highload/internal/repository/postgres [no test files] +? highload/internal/repository/redis [no test files] +? highload/internal/security [no test files] +? highload/internal/service [no test files] +? highload/internal/storage [no test files] +? highload/internal/storage/s3 [no test files] +? highload/internal/transport/http [no test files] +? highload/internal/transport/http/handler [no test files] +? highload/internal/transport/http/httputils [no test files] +? highload/internal/transport/http/middleware [no test files] +=== RUN TestAttachmentCleanupPagesAndCutoff +--- PASS: TestAttachmentCleanupPagesAndCutoff (0.00s) +=== RUN TestAttachmentCleanupFaults +=== RUN TestAttachmentCleanupFaults/list +=== RUN TestAttachmentCleanupFaults/exists +=== RUN TestAttachmentCleanupFaults/delete +=== RUN TestAttachmentCleanupFaults/list_timeout +=== RUN TestAttachmentCleanupFaults/pass_timeout +=== RUN TestAttachmentCleanupFaults/canceled +--- PASS: TestAttachmentCleanupFaults (0.00s) + --- PASS: TestAttachmentCleanupFaults/list (0.00s) + --- PASS: TestAttachmentCleanupFaults/exists (0.00s) + --- PASS: TestAttachmentCleanupFaults/delete (0.00s) + --- PASS: TestAttachmentCleanupFaults/list_timeout (0.00s) + --- PASS: TestAttachmentCleanupFaults/pass_timeout (0.00s) + --- PASS: TestAttachmentCleanupFaults/canceled (0.00s) +=== RUN TestAttachmentCleanupS3InvalidNextToken +=== RUN TestAttachmentCleanupS3InvalidNextToken/missing +=== RUN TestAttachmentCleanupS3InvalidNextToken/empty +=== RUN TestAttachmentCleanupS3InvalidNextToken/repeat +--- PASS: TestAttachmentCleanupS3InvalidNextToken (0.00s) + --- PASS: TestAttachmentCleanupS3InvalidNextToken/missing (0.00s) + --- PASS: TestAttachmentCleanupS3InvalidNextToken/empty (0.00s) + --- PASS: TestAttachmentCleanupS3InvalidNextToken/repeat (0.00s) +=== RUN TestAttachmentTxLockAndRollback +=== RUN TestAttachmentTxLockAndRollback/callback_error + attachment_tx_test.go:160: INSERT ran at read committed, rollback left zero rows and released the lock +=== RUN TestAttachmentTxLockAndRollback/context_canceled + attachment_tx_test.go:160: INSERT ran at read committed, rollback left zero rows and released the lock +--- PASS: TestAttachmentTxLockAndRollback (0.10s) + --- PASS: TestAttachmentTxLockAndRollback/callback_error (0.01s) + --- PASS: TestAttachmentTxLockAndRollback/context_canceled (0.00s) +=== RUN TestAttachmentTxHTTP +=== RUN TestAttachmentTxHTTP/s3_unavailable +2026/11/15 11:47:25 ERROR request failed error="service unavailable: object storage unavailable: operation error S3: PutObject, https response error StatusCode: 503, RequestID: , HostID: , api error SlowDown: Synthetic outage" request_id=4410ba4d-b703-4050-beba-e9c9ddb4942c status=503 code=unavailable + attachment_tx_test.go:288: direct handler HTTP 503; committed rows=0; DELETE=0 +=== RUN TestAttachmentTxHTTP/insert_after_put +2026/11/15 11:47:25 ERROR request failed error="server error: database error: ERROR: new row for relation \"attachments\" violates check constraint \"attachments_filename_check1\" (SQLSTATE 23514)" request_id=55830874-e46a-4622-ba4a-24bd2712a3d7 status=500 code=internal_error + attachment_tx_test.go:288: direct handler HTTP 500; committed rows=0; DELETE=0 +=== RUN TestAttachmentTxHTTP/commit_after_insert +2026/11/15 11:47:25 ERROR request failed error="server error: database error: ERROR: synthetic deferred failure (SQLSTATE P0001)" request_id=61ebcff4-daf1-43ed-98b0-b305c659a3e4 status=500 code=internal_error + attachment_tx_test.go:288: direct handler HTTP 500; committed rows=0; DELETE=0 +=== RUN TestAttachmentTxHTTP/committed_upload_and_get + attachment_tx_test.go:315: HTTP 201, committed rows=1, GET metadata/content=200, SHA-256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +--- PASS: TestAttachmentTxHTTP (0.23s) + --- PASS: TestAttachmentTxHTTP/s3_unavailable (0.06s) + --- PASS: TestAttachmentTxHTTP/insert_after_put (0.05s) + --- PASS: TestAttachmentTxHTTP/commit_after_insert (0.05s) + --- PASS: TestAttachmentTxHTTP/committed_upload_and_get (0.06s) +=== RUN TestStorageAuthHandlerRejectsJSONBody +--- PASS: TestStorageAuthHandlerRejectsJSONBody (0.12s) +=== RUN TestPostgresOperationsHaveBoundedWaits + budgets_test.go:96: budget_policy=provisional_3000ms; teacher has no numeric threshold; approval remains pending +=== RUN TestPostgresOperationsHaveBoundedWaits/blocked_query/application_budget +2026/11/15 11:47:27 ERROR request failed error="service unavailable: database unavailable: timeout: context deadline exceeded" request_id=af0ade91-2c05-446b-881f-5085da4a638a status=503 code=unavailable +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=503 duration=1.002822167s request_id=af0ade91-2c05-446b-881f-5085da4a638a + budgets_test.go:130: mode=blocked_query scenario=application_budget elapsed_ms=1003.015 caller_deadline_ms=5000 budget_ms=3000 caller_canceled=false +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=200 duration=18.338542ms request_id=2bccce82-b04e-48c0-b75d-10a5ea07e12d +=== RUN TestPostgresOperationsHaveBoundedWaits/blocked_query/shorter_caller_deadline +2026/11/15 11:47:27 ERROR request failed error="service unavailable: database unavailable: timeout: context deadline exceeded" request_id=767c833b-b621-4e15-b2ca-18b9a3165ef5 status=503 code=unavailable +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=503 duration=150.605583ms request_id=767c833b-b621-4e15-b2ca-18b9a3165ef5 + budgets_test.go:130: mode=blocked_query scenario=shorter_caller_deadline elapsed_ms=150.686 caller_deadline_ms=150 budget_ms=3000 caller_canceled=true +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=200 duration=5.120542ms request_id=71588141-78a7-4c05-b5f4-92282e58feb7 +=== RUN TestPostgresOperationsHaveBoundedWaits/blocked_query/released_before_budget +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=200 duration=103.494542ms request_id=e42ec7f1-3162-4691-ad92-8b3b6b78821f + budgets_test.go:130: mode=blocked_query scenario=released_before_budget elapsed_ms=103.631 caller_deadline_ms=5000 budget_ms=3000 caller_canceled=false +2026/11/15 11:47:27 INFO http request method=GET path=/api/v1/users/me status=200 duration=845.375µs request_id=8c8f48e9-e829-4355-8d25-987223cf4d2e +=== RUN TestPostgresOperationsHaveBoundedWaits/exhausted_pool/application_budget +2026/11/15 11:47:29 ERROR request failed error="service unavailable: database unavailable: context deadline exceeded" request_id=6c0366cf-55fe-42d7-9f74-f64646887a94 status=503 code=unavailable +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=503 duration=1.003118167s request_id=6c0366cf-55fe-42d7-9f74-f64646887a94 + budgets_test.go:130: mode=exhausted_pool scenario=application_budget elapsed_ms=1003.262 caller_deadline_ms=5000 budget_ms=3000 caller_canceled=false +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=200 duration=15.015958ms request_id=9b2a8a8d-e656-4c03-9bd0-376a97d1521e +=== RUN TestPostgresOperationsHaveBoundedWaits/exhausted_pool/shorter_caller_deadline +2026/11/15 11:47:29 ERROR request failed error="service unavailable: database unavailable: context deadline exceeded" request_id=c5e50fb3-3d35-4b17-9d06-552520839462 status=503 code=unavailable +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=503 duration=150.302291ms request_id=c5e50fb3-3d35-4b17-9d06-552520839462 + budgets_test.go:130: mode=exhausted_pool scenario=shorter_caller_deadline elapsed_ms=150.378 caller_deadline_ms=150 budget_ms=3000 caller_canceled=true +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=200 duration=4.709083ms request_id=c6d568e7-75c2-473f-83db-d4feb7004ad7 +=== RUN TestPostgresOperationsHaveBoundedWaits/exhausted_pool/released_before_budget +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=200 duration=104.233375ms request_id=e5c4c586-9fd8-499c-9478-66077e6d72b4 + budgets_test.go:130: mode=exhausted_pool scenario=released_before_budget elapsed_ms=104.332 caller_deadline_ms=5000 budget_ms=3000 caller_canceled=false +2026/11/15 11:47:29 INFO http request method=GET path=/api/v1/users/me status=200 duration=1.092834ms request_id=140de8ac-41b5-4d0d-9cc7-d6f4d1bc4b18 +--- PASS: TestPostgresOperationsHaveBoundedWaits (4.02s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/blocked_query/application_budget (1.24s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/blocked_query/shorter_caller_deadline (0.40s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/blocked_query/released_before_budget (0.33s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/exhausted_pool/application_budget (1.27s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/exhausted_pool/shorter_caller_deadline (0.42s) + --- PASS: TestPostgresOperationsHaveBoundedWaits/exhausted_pool/released_before_budget (0.36s) +=== RUN TestPostgresChatCancellationBeforeCommitIsAtomic +--- PASS: TestPostgresChatCancellationBeforeCommitIsAtomic (0.35s) +=== RUN TestErrorDiagnosticsRecordedOnceAndPublicResponseIsSafe +--- PASS: TestErrorDiagnosticsRecordedOnceAndPublicResponseIsSafe (0.00s) +=== RUN TestErrorDatabaseCategoryReachesHTTP +=== RUN TestErrorDatabaseCategoryReachesHTTP/no_rows +=== RUN TestErrorDatabaseCategoryReachesHTTP/duplicate +=== RUN TestErrorDatabaseCategoryReachesHTTP/eof +=== RUN TestErrorDatabaseCategoryReachesHTTP/deadline +=== RUN TestErrorDatabaseCategoryReachesHTTP/unexpected_eof +=== RUN TestErrorDatabaseCategoryReachesHTTP/closed_connection +=== RUN TestErrorDatabaseCategoryReachesHTTP/connection_refused +=== RUN TestErrorDatabaseCategoryReachesHTTP/database_dns_missing +=== RUN TestErrorDatabaseCategoryReachesHTTP/sql_syntax +=== RUN TestErrorDatabaseCategoryReachesHTTP/unexpected +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_53300 +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P03 +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_08006 +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_08001 +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P01 +=== RUN TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P02 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_no_rows +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_duplicate +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_eof +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_deadline +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_unexpected_eof +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_closed_connection +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_connection_refused +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_database_dns_missing +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sql_syntax +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_unexpected +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_53300 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P03 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_08006 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_08001 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P01 +=== RUN TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P02 +--- PASS: TestErrorDatabaseCategoryReachesHTTP (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/no_rows (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/duplicate (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/eof (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/deadline (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/unexpected_eof (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/closed_connection (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/connection_refused (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/database_dns_missing (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sql_syntax (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/unexpected (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_53300 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P03 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_08006 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_08001 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P01 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/sqlstate_57P02 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_no_rows (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_duplicate (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_eof (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_deadline (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_unexpected_eof (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_closed_connection (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_connection_refused (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_database_dns_missing (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sql_syntax (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_unexpected (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_53300 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P03 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_08006 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_08001 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P01 (0.00s) + --- PASS: TestErrorDatabaseCategoryReachesHTTP/wrapped_sqlstate_57P02 (0.00s) +=== RUN TestErrorUnavailableSQLLoginKeepsCategory +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/eof +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/deadline +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/unexpected_eof +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/closed_connection +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/connection_refused +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/database_dns_missing +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_53300 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P03 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_08006 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_08001 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P01 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P02 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_eof +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_deadline +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_unexpected_eof +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_closed_connection +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_connection_refused +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_database_dns_missing +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_53300 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P03 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_08006 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_08001 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P01 +=== RUN TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P02 +--- PASS: TestErrorUnavailableSQLLoginKeepsCategory (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/eof (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/deadline (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/unexpected_eof (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/closed_connection (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/connection_refused (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/database_dns_missing (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_53300 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P03 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_08006 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_08001 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P01 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/sqlstate_57P02 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_eof (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_deadline (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_unexpected_eof (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_closed_connection (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_connection_refused (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_database_dns_missing (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_53300 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P03 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_08006 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_08001 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P01 (0.00s) + --- PASS: TestErrorUnavailableSQLLoginKeepsCategory/wrapped_sqlstate_57P02 (0.00s) +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/no_rows +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/eof +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/deadline +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/unexpected_eof +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/closed_connection +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/connection_refused +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/database_dns_missing +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sql_syntax +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/unexpected +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_53300 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P03 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_08006 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_08001 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P01 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P02 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_no_rows +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_eof +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_deadline +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_unexpected_eof +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_closed_connection +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_connection_refused +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_database_dns_missing +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sql_syntax +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_unexpected +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_53300 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P03 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_08006 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_08001 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P01 +=== RUN TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P02 +--- PASS: TestErrorChatMemberLookupKeepsOperationMeaning (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/no_rows (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/eof (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/deadline (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/unexpected_eof (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/closed_connection (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/connection_refused (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/database_dns_missing (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sql_syntax (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/unexpected (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_53300 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P03 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_08006 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_08001 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P01 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/sqlstate_57P02 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_no_rows (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_eof (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_deadline (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_unexpected_eof (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_closed_connection (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_connection_refused (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_database_dns_missing (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sql_syntax (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_unexpected (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_53300 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P03 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_08006 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_08001 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P01 (0.00s) + --- PASS: TestErrorChatMemberLookupKeepsOperationMeaning/wrapped_sqlstate_57P02 (0.00s) +=== RUN TestErrorMissingBasicPrecedesForbiddenBody +--- PASS: TestErrorMissingBasicPrecedesForbiddenBody (0.00s) +=== RUN TestRedisLogoutPersistenceDiagnostic + logout_diagnostic_test.go:133: logout samples=51 pool_wait_count=0 wait_ns=0 pool_timeouts=0 final_busy=0 +--- PASS: TestRedisLogoutPersistenceDiagnostic (13.24s) +=== RUN TestMessageAttachmentsSingleIDServiceRegression +=== RUN TestMessageAttachmentsSingleIDServiceRegression/single_attachment_without_panic +=== RUN TestMessageAttachmentsSingleIDServiceRegression/identical_uppercase_body_replay +--- PASS: TestMessageAttachmentsSingleIDServiceRegression (0.00s) + --- PASS: TestMessageAttachmentsSingleIDServiceRegression/single_attachment_without_panic (0.00s) + --- PASS: TestMessageAttachmentsSingleIDServiceRegression/identical_uppercase_body_replay (0.00s) +=== RUN TestMessageAttachmentsTextCompatibility +--- PASS: TestMessageAttachmentsTextCompatibility (0.10s) +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_0 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_1 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_2 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_3 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_4 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_5 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_6 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_7 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_8 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_9 +=== RUN TestMessageAttachmentsOnlyAndOrderZeroToTen/count_10 +--- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen (0.64s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_0 (0.08s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_1 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_2 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_3 (0.05s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_4 (0.05s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_5 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_6 (0.05s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_7 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_8 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_9 (0.06s) + --- PASS: TestMessageAttachmentsOnlyAndOrderZeroToTen/count_10 (0.05s) +=== RUN TestMessageAttachmentsInvalidReferencesAreAtomic +=== RUN TestMessageAttachmentsInvalidReferencesAreAtomic/unknown_id +=== RUN TestMessageAttachmentsInvalidReferencesAreAtomic/attachment_from_other_chat +=== RUN TestMessageAttachmentsInvalidReferencesAreAtomic/foreign_chat +--- PASS: TestMessageAttachmentsInvalidReferencesAreAtomic (0.21s) + --- PASS: TestMessageAttachmentsInvalidReferencesAreAtomic/unknown_id (0.05s) + --- PASS: TestMessageAttachmentsInvalidReferencesAreAtomic/attachment_from_other_chat (0.05s) + --- PASS: TestMessageAttachmentsInvalidReferencesAreAtomic/foreign_chat (0.10s) +=== RUN TestMessageAttachmentsInvalidListLeavesNoRows +=== RUN TestMessageAttachmentsInvalidListLeavesNoRows/duplicate +=== RUN TestMessageAttachmentsInvalidListLeavesNoRows/eleven +--- PASS: TestMessageAttachmentsInvalidListLeavesNoRows (0.11s) + --- PASS: TestMessageAttachmentsInvalidListLeavesNoRows/duplicate (0.05s) + --- PASS: TestMessageAttachmentsInvalidListLeavesNoRows/eleven (0.05s) +=== RUN TestMessageAttachmentsIdempotencyIncludesListAndOrder +--- PASS: TestMessageAttachmentsIdempotencyIncludesListAndOrder (0.12s) +=== RUN TestMessageAttachmentsConcurrentReplayDoesNotMultiplyLinks +--- PASS: TestMessageAttachmentsConcurrentReplayDoesNotMultiplyLinks (0.08s) +=== RUN TestMessageAttachmentsLinkFailureRollsBackEverything +--- PASS: TestMessageAttachmentsLinkFailureRollsBackEverything (0.07s) +=== RUN TestPasswordRoundTripAndRandomSalt +--- PASS: TestPasswordRoundTripAndRandomSalt (0.54s) +=== RUN TestPasswordDoesNotTruncateLongUTF8 +--- PASS: TestPasswordDoesNotTruncateLongUTF8 (0.22s) +=== RUN TestPasswordWhitespaceIsSignificant +--- PASS: TestPasswordWhitespaceIsSignificant (0.15s) +=== RUN TestMalformedPasswordHashRejectedWithoutPanic +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/plaintext +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/wrong_algorithm +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/huge_iterations +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/missing_param +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/hash_empty +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/hash_wrong_size +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/salt_wrong_size +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/salt_empty +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/extra_field +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/zero_iterations +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/negative +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/non_numeric +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/empty +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/missing_version +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/tiny_memory +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/huge_threads +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/zero_threads +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/huge_memory +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/hash_bad_base64 +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/wrong_version +=== RUN TestMalformedPasswordHashRejectedWithoutPanic/salt_bad_base64 +--- PASS: TestMalformedPasswordHashRejectedWithoutPanic (0.07s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/plaintext (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/wrong_algorithm (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/huge_iterations (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/missing_param (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/hash_empty (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/hash_wrong_size (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/salt_wrong_size (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/salt_empty (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/extra_field (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/zero_iterations (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/negative (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/non_numeric (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/empty (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/missing_version (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/tiny_memory (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/huge_threads (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/zero_threads (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/huge_memory (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/hash_bad_base64 (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/wrong_version (0.00s) + --- PASS: TestMalformedPasswordHashRejectedWithoutPanic/salt_bad_base64 (0.00s) +=== RUN TestRedisAddressIsAnAddressNotAFile +--- PASS: TestRedisAddressIsAnAddressNotAFile (0.00s) +=== RUN TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout +=== RUN TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/local_not_fsynced +=== RUN TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/only_replica_fsynced +=== RUN TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/malformed_reply +=== RUN TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/command_failure +--- PASS: TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout (0.83s) + --- PASS: TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/local_not_fsynced (0.21s) + --- PASS: TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/only_replica_fsynced (0.21s) + --- PASS: TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/malformed_reply (0.21s) + --- PASS: TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout/command_failure (0.21s) +=== RUN TestRedisSessionHasTTLAndReadsDoNotExtendIt +--- PASS: TestRedisSessionHasTTLAndReadsDoNotExtendIt (0.13s) +=== RUN TestRedisExpiredAndUnknownSessionsAreNotFound +--- PASS: TestRedisExpiredAndUnknownSessionsAreNotFound (0.36s) +=== RUN TestRedisLogoutOnlyRemovesSelectedSession +--- PASS: TestRedisLogoutOnlyRemovesSelectedSession (1.03s) +=== RUN TestRedisSessionsSurviveActualRestartWithoutExtendingTTL +--- PASS: TestRedisSessionsSurviveActualRestartWithoutExtendingTTL (7.59s) +=== RUN TestRedisActiveSessionSurvivesRestartIndependentlyOfLogout +--- PASS: TestRedisActiveSessionSurvivesRestartIndependentlyOfLogout (6.18s) +=== RUN TestRedisUnavailableKeepsUnavailableCategory +=== RUN TestRedisUnavailableKeepsUnavailableCategory/create +redis: 2026/11/15 11:48:02 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +=== RUN TestRedisUnavailableKeepsUnavailableCategory/read +redis: 2026/11/15 11:48:02 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +=== RUN TestRedisUnavailableKeepsUnavailableCategory/logout +redis: 2026/11/15 11:48:03 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +redis: 2026/11/15 11:48:03 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +--- PASS: TestRedisUnavailableKeepsUnavailableCategory (1.71s) + --- PASS: TestRedisUnavailableKeepsUnavailableCategory/create (0.42s) + --- PASS: TestRedisUnavailableKeepsUnavailableCategory/read (0.41s) + --- PASS: TestRedisUnavailableKeepsUnavailableCategory/logout (0.41s) +=== RUN TestRedisCriticalOutageHTTPAndReadiness +=== RUN TestRedisCriticalOutageHTTPAndReadiness/liveness +=== RUN TestRedisCriticalOutageHTTPAndReadiness/readiness +redis: 2026/11/15 11:48:04 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +=== RUN TestRedisCriticalOutageHTTPAndReadiness/protected_request +redis: 2026/11/15 11:48:04 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp: connection refused +--- PASS: TestRedisCriticalOutageHTTPAndReadiness (0.97s) + --- PASS: TestRedisCriticalOutageHTTPAndReadiness/liveness (0.00s) + --- PASS: TestRedisCriticalOutageHTTPAndReadiness/readiness (0.41s) + --- PASS: TestRedisCriticalOutageHTTPAndReadiness/protected_request (0.41s) +=== RUN TestRedisPublicHTTPContractIsUnchanged +=== RUN TestRedisPublicHTTPContractIsUnchanged/teacher_smoke +=== RUN TestRedisPublicHTTPContractIsUnchanged/auth_normal +--- PASS: TestRedisPublicHTTPContractIsUnchanged (7.92s) + --- PASS: TestRedisPublicHTTPContractIsUnchanged/teacher_smoke (1.58s) + --- PASS: TestRedisPublicHTTPContractIsUnchanged/auth_normal (6.28s) +=== RUN TestRedisHTTPFullRestartKeepsDeadlineAndRevocation +=== RUN TestRedisHTTPFullRestartKeepsDeadlineAndRevocation/active_only +=== RUN TestRedisHTTPFullRestartKeepsDeadlineAndRevocation/active_and_revoked +--- PASS: TestRedisHTTPFullRestartKeepsDeadlineAndRevocation (33.04s) + --- PASS: TestRedisHTTPFullRestartKeepsDeadlineAndRevocation/active_only (15.91s) + --- PASS: TestRedisHTTPFullRestartKeepsDeadlineAndRevocation/active_and_revoked (17.13s) +=== RUN TestStoragePasswordIsKDFAndDuplicateCannotOverwrite +--- PASS: TestStoragePasswordIsKDFAndDuplicateCannotOverwrite (0.38s) +=== RUN TestStorageSessionHashAndReconnectPreserveDeadline +--- PASS: TestStorageSessionHashAndReconnectPreserveDeadline (0.20s) +=== RUN TestStorageWrongCredentialsNeverCreateSession +--- PASS: TestStorageWrongCredentialsNeverCreateSession (0.18s) +=== RUN TestStorageRevocationIsPersistentAndSessionSpecific +--- PASS: TestStorageRevocationIsPersistentAndSessionSpecific (1.35s) +=== RUN TestStorageExpiredSessionCannotRevive +--- PASS: TestStorageExpiredSessionCannotRevive (0.25s) +=== RUN TestStorageShortTTLExpiresAndReLoginWorks +--- PASS: TestStorageShortTTLExpiresAndReLoginWorks (2.43s) +=== RUN TestStorageChatRollbackAfterMemberInsertFailure +--- PASS: TestStorageChatRollbackAfterMemberInsertFailure (0.14s) +=== RUN TestSQLUnavailableLoginStopsBeforeSessionCreation +--- PASS: TestSQLUnavailableLoginStopsBeforeSessionCreation (0.00s) +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#01 +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#02 +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#03 +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#04 +=== RUN TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#05 +--- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#01 (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#02 (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#03 (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#04 (0.00s) + --- PASS: TestMiddlewareMalformedAuthWritesExactlyOneError/malformed#05 (0.00s) +=== RUN TestStorageHTTPAuthRoutingAndLogout +--- PASS: TestStorageHTTPAuthRoutingAndLogout (0.86s) +PASS +ok highload/student-tests/auth_tests 86.386s diff --git a/evidence/lab4-grade5/migrate.txt b/evidence/lab4-grade5/migrate.txt new file mode 100644 index 00000000..482402cd --- /dev/null +++ b/evidence/lab4-grade5/migrate.txt @@ -0,0 +1,7 @@ +migrate-1 | {"time":"2026-11-15T08:47:02.287184423Z","level":"INFO","msg":"migration applied","file":"00001_init_schema.sql","version":1,"duration_seconds":0.010336709} +migrate-1 | {"time":"2026-11-15T08:47:02.287565965Z","level":"INFO","msg":"migration applied","file":"00002_add_created_at_defaults.sql","version":2,"duration_seconds":0.000999708} +migrate-1 | {"time":"2026-11-15T08:47:02.28757859Z","level":"INFO","msg":"migration applied","file":"00003_drop_session_table.sql","version":3,"duration_seconds":0.003952625} +migrate-1 | {"time":"2026-11-15T08:47:02.287583923Z","level":"INFO","msg":"migration applied","file":"00004_message_idempotency.sql","version":4,"duration_seconds":0.0022235} +migrate-1 | {"time":"2026-11-15T08:47:02.287588381Z","level":"INFO","msg":"migration applied","file":"00005_attachments.sql","version":5,"duration_seconds":0.008834} +migrate-1 | {"time":"2026-11-15T08:47:02.287592048Z","level":"INFO","msg":"migration applied","file":"00006_allow_attachment_only_messages.sql","version":6,"duration_seconds":0.0015365} +migrate-1 | {"time":"2026-11-15T08:47:02.287596715Z","level":"INFO","msg":"migrations completed","applied":6,"duration_seconds":0.060165208} diff --git a/evidence/lab4-grade5/persistence.txt b/evidence/lab4-grade5/persistence.txt new file mode 100644 index 00000000..b53f5ebf --- /dev/null +++ b/evidence/lab4-grade5/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18096" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab4-grade5/pixel-limit.json b/evidence/lab4-grade5/pixel-limit.json new file mode 100644 index 00000000..52e7d9de --- /dev/null +++ b/evidence/lab4-grade5/pixel-limit.json @@ -0,0 +1,6 @@ +{ + "width": 6500, + "height": 6200, + "encoded_bytes": 39249, + "upload_status": 413 +} diff --git a/evidence/lab4-grade5/reconcile.json b/evidence/lab4-grade5/reconcile.json new file mode 100644 index 00000000..78727722 --- /dev/null +++ b/evidence/lab4-grade5/reconcile.json @@ -0,0 +1,333 @@ +{ + "faults": { + "injected": [ + "delete PostgreSQL row", + "delete S3 object", + "S3 connection refused" + ], + "dry_run": { + "dry_run": true, + "duration_ms": 31, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 4, + "checked_in_db": 3, + "busy": 0, + "orphan_candidates": 1, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 3, + "s3_head_attempts": 3, + "missing_objects": 1, + "missing_attachment_ids": [ + "0d3b9f77-50a9-4d31-90fa-2d25b4cd613e" + ] + }, + "wall_ms": 607.98 + }, + "bad_backup_rejected": { + "dry_run": false, + "duration_ms": 8, + "complete": false, + "wall_ms": 685.91 + }, + "restore_dry_run": { + "dry_run": true, + "duration_ms": 14, + "complete": true, + "restore": "would_restore", + "wall_ms": 603.21 + }, + "restore": { + "dry_run": false, + "duration_ms": 22, + "complete": true, + "restore": "restored", + "wall_ms": 431.95 + }, + "repeat_restore": { + "dry_run": false, + "duration_ms": 12, + "complete": true, + "restore": "already_present", + "wall_ms": 430.94 + }, + "conditional_existing_put_status": 412, + "apply": { + "dry_run": false, + "duration_ms": 190, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 5, + "checked_in_db": 3, + "busy": 0, + "orphan_candidates": 1, + "objects_deleted": 1 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 3, + "s3_head_attempts": 3, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 689.99 + }, + "repeat": { + "dry_run": false, + "duration_ms": 59, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 3, + "checked_in_db": 2, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 3, + "s3_head_attempts": 3, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 653.52 + }, + "s3_outage": { + "dry_run": true, + "duration_ms": 107, + "complete": false, + "missing_scan": { + "db_pages": 0, + "rows_scanned": 0, + "s3_head_attempts": 0, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 869.21 + }, + "fresh_preserved": true, + "linked_preserved": true, + "unlinked_preserved": true, + "sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "message_and_metadata_preserved": true, + "outsider_status": 404 + }, + "cost": [ + { + "synthetic_pairs": 30, + "measured_pairs": 33, + "runs": [ + { + "dry_run": true, + "duration_ms": 82, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 33, + "checked_in_db": 33, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 33, + "s3_head_attempts": 33, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 1791.83 + }, + { + "dry_run": true, + "duration_ms": 100, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 33, + "checked_in_db": 33, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 33, + "s3_head_attempts": 33, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 588.31 + }, + { + "dry_run": true, + "duration_ms": 32, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 33, + "checked_in_db": 33, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 1, + "rows_scanned": 33, + "s3_head_attempts": 33, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 572.91 + } + ], + "median_scan_ms": 82 + }, + { + "synthetic_pairs": 230, + "measured_pairs": 233, + "runs": [ + { + "dry_run": true, + "duration_ms": 227, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 233, + "checked_in_db": 233, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 2, + "rows_scanned": 233, + "s3_head_attempts": 233, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 826.69 + }, + { + "dry_run": true, + "duration_ms": 132, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 233, + "checked_in_db": 233, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 2, + "rows_scanned": 233, + "s3_head_attempts": 233, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 591.24 + }, + { + "dry_run": true, + "duration_ms": 147, + "complete": true, + "orphan_scan": { + "s3_pages": 1, + "objects_scanned": 233, + "checked_in_db": 233, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 2, + "rows_scanned": 233, + "s3_head_attempts": 233, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 699.16 + } + ], + "median_scan_ms": 147 + }, + { + "synthetic_pairs": 1050, + "measured_pairs": 1053, + "runs": [ + { + "dry_run": true, + "duration_ms": 629, + "complete": true, + "orphan_scan": { + "s3_pages": 2, + "objects_scanned": 1053, + "checked_in_db": 1053, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 6, + "rows_scanned": 1053, + "s3_head_attempts": 1053, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 1885.23 + }, + { + "dry_run": true, + "duration_ms": 587, + "complete": true, + "orphan_scan": { + "s3_pages": 2, + "objects_scanned": 1053, + "checked_in_db": 1053, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 6, + "rows_scanned": 1053, + "s3_head_attempts": 1053, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 1125.28 + }, + { + "dry_run": true, + "duration_ms": 3070, + "complete": true, + "orphan_scan": { + "s3_pages": 2, + "objects_scanned": 1053, + "checked_in_db": 1053, + "busy": 0, + "orphan_candidates": 0, + "objects_deleted": 0 + }, + "missing_scan": { + "db_pages": 6, + "rows_scanned": 1053, + "s3_head_attempts": 1053, + "missing_objects": 0, + "missing_attachment_ids": [] + }, + "wall_ms": 4871.64 + } + ], + "median_scan_ms": 629 + } + ] +} diff --git a/evidence/lab4-grade5/s3-security.json b/evidence/lab4-grade5/s3-security.json new file mode 100644 index 00000000..9af00130 --- /dev/null +++ b/evidence/lab4-grade5/s3-security.json @@ -0,0 +1,68 @@ +{ + "s3_api_bucket_scope_and_iam_denials": { + "detail": "", + "elapsed_seconds": 9.24, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_api_delete_effect": { + "detail": "", + "elapsed_seconds": 2.247, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_api_effective_policy": { + "detail": "", + "elapsed_seconds": 2.677, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_bootstrap_preserves_object_and_volume": { + "detail": "", + "elapsed_seconds": 4.152, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_console_closed_on_shared_network": { + "detail": "", + "elapsed_seconds": 1.491, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_fixture_cleanup": { + "detail": "", + "elapsed_seconds": 4.488, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_restart_preserves_original_bytes": { + "detail": "", + "elapsed_seconds": 6.478, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_setup_and_scenarios": { + "detail": "", + "elapsed_seconds": 33.571, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_two_replicas_after_restart": { + "detail": "", + "elapsed_seconds": 0.718, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_two_replicas_before": { + "detail": "", + "elapsed_seconds": 1.003, + "requirement": "requested_operational_check", + "status": "PASS" + }, + "s3_unsigned_known_object_403": { + "detail": "", + "elapsed_seconds": 0.453, + "requirement": "requested_operational_check", + "status": "PASS" + } +} diff --git a/evidence/lab4-grade5/smoke.txt b/evidence/lab4-grade5/smoke.txt new file mode 100644 index 00000000..384e509c --- /dev/null +++ b/evidence/lab4-grade5/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://127.0.0.1:18097" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab4-grade5/source.json b/evidence/lab4-grade5/source.json new file mode 100644 index 00000000..54a89cb2 --- /dev/null +++ b/evidence/lab4-grade5/source.json @@ -0,0 +1,100 @@ +{ + "files": { + "Dockerfile": "17e5c9fa261dbe5395791213f605787824957da6567586ce990e44e3fd5a1236", + "Dockerfile.migrate": "0ab04cba4c2bb033c4d7c7d3a5984d8a27e8f355ff90e6023c3b7912d2211dec", + "cmd/api/main.go": "e9568a3a1098b01fded10cdc502dac652bf6a3a5761914c6860c9f45b56ed269", + "cmd/api/poolstats.go": "75c0ca950b85eaee3c0802e3001c61ba6436874b3be12cd38a5feb907b91528e", + "cmd/attachment-cleanup/main.go": "2658e84d6a062552303e04e41e45bd0b5fa1f6a75e90158f616d60fedbef3b3d", + "cmd/migrate/main.go": "6e9d413a86f53cf5f2e83d5ed0308af9b2c33e69fcb3d061b19bf480e352f257", + "cmd/migrate/main_test.go": "a88f2252d70caf644904331182dc3afd4df48b825a0329a82482798ec90b5d46", + "cmd/migrate/postgres_test.go": "37401cd60657af560d52e727b08861f444ab18355d9decd9ceeeddb009d079ca", + "cmd/migrate/roundtrip_test.go": "25b3425447b1dbafec83db2e5a7fc8172031908d5f5d57ce7b0d36a97850f8fa", + "compose.yaml": "e67836432e988a1035a91ed028471e801fcf0827fb6ff4070f1e583abc06d787", + "db/migrations/00001_init_schema.sql": "48db014273f78be0f6eb97fc521cccc54f851dc99e3688d480d3a6ff7b9b6bbe", + "db/migrations/00002_add_created_at_defaults.sql": "e0680838294b93f916d727a3a20d83aca7c3f74077202c9696b1fc73cda14649", + "db/migrations/00003_drop_session_table.sql": "1b666cee95b9eb1210b0a303de95994bd3c24628e9d9fdb1e1f3318edb756c3d", + "db/migrations/00004_message_idempotency.sql": "e7b3dd5a056a702dfde7acba6666430477db0c7e03ae03fb270e145d6303b9b2", + "db/migrations/00005_attachments.sql": "033d8884cecb50b5067453a97f2d7c51e51331a1911d9ddf8687437afa59443c", + "db/migrations/00006_allow_attachment_only_messages.sql": "ad56ecd0e8b88e0665cdc144aaa5d934ea7f48047cd702ce6f8832b6cf0bded3", + "db/migrations/embed.go": "a42e52ab05bc89b557a5bc6b3f0215883d99c4a039a8b32d6c19e66b2f769508", + "db/queries/attachments.sql": "cc030be1e7ed01d5895d37f0cf10d66821ef2c30dfd7fba4fb155488b8c96ebc", + "db/queries/chat_members.sql": "c80b6e50c321ff94317545ea2f9fdee1dfd69f564d8638f7b111c8ec091ef03a", + "db/queries/chats.sql": "8c52c762da78eb3a3eb7402b17ec87bd7fc2e1387f6ddb8e297b4f2d75beca7b", + "db/queries/message_idempotence.sql": "34259e4beee9cc07de4dec95c1d588222c7d756d02ac6564b54d6dbec36678db", + "db/queries/messages.sql": "f6f6382f27d698b1c2af2305d45699226ef10806b938b3dca78ec9087a622b7b", + "db/queries/messages_attachments.sql": "753e5b46dbe94b1043fbe19c83850e697feef7ea9ccc348496b71cb6f9906ddb", + "db/queries/users.sql": "0d73a6d46f4d4a1b0af37b750a208c07a38249f76974b41748f32c1949f990df", + "go.mod": "0dbf79ae49966e763868e20b453addaecbb74085dd7b0579271f0839ca8a17cd", + "go.sum": "30264a0cbc88f40ac455abb621624a54aa956bedd390c07deef4f67812efc982", + "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", + "infra/haproxy.cfg": "895a178e7354de61a5d40a2fe18b7b7c6d1d3d5faf0e5a5faa72783e7b983cb8", + "infra/s3/bootstrap.sh": "baa4b0b6fa4bd2a76b194b672ef2ca8575858ef6abc7ddfbede3eec3534bc8ce", + "internal/config/config.go": "142318dac05e3070358e7642c4b09813fe6bdb4d1b80e1154954d5e4e5f3fa59", + "internal/database/postgres.go": "3925c0bc9ae01bff9a37138a0dd19b38074bc2a70536b3a148f568b952516a21", + "internal/database/redis.go": "5b85a60bb0d1cb156c61bcd3c518c41c955f53c1d12da265b5e3a5dd7a397602", + "internal/database/sqlc/attachments.sql.go": "607a42ff362b4e69880a58b7eb989a0aac5d8e1b39df005dfae0c607dec659c5", + "internal/database/sqlc/chat_members.sql.go": "9e8cd177f6f888c72d19857fd4748fbb2020770c1af772a463fe3d1b813b37ee", + "internal/database/sqlc/chats.sql.go": "3dccf4b0ffa918ec5a3b66026b3dab4027456b499486359881000d184944802c", + "internal/database/sqlc/db.go": "3ca71a7f007a39987cae838da64303419dd804d2d1b6227676f47177971d76cf", + "internal/database/sqlc/message_idempotence.sql.go": "ebd31e6f369483eed8d6978dab1687552eed742a612b5ee4090499fe8a48a73f", + "internal/database/sqlc/messages.sql.go": "9836433d9645e6dd3bd66273ce15a53691f79093016c416fb24ffd085e62831e", + "internal/database/sqlc/messages_attachments.sql.go": "821d863a01079edae53805769821f1dc5eb3e32620ce61efd7326bb0253d2204", + "internal/database/sqlc/models.go": "df06356e705031dfc6226efd736f432853e026c26c30424675b99179819d422e", + "internal/database/sqlc/users.sql.go": "b954abd5b5964df39c20d857a60105513af84c8df68ce1ac9421da541a7800f4", + "internal/model/attachment.go": "877de511f5996c263c63c3e406ed67ee6609833f4946074594e1d438f8dce53a", + "internal/model/chat.go": "e0213f8360ea106b3627aa50cc87710153ff8e353475b42c2becdb9a32592660", + "internal/model/message.go": "4cebc4963e2eb104e85a1df6e8a93d6745c9ae42425df2d04bf6319e1d17b12d", + "internal/model/session.go": "b455d730e4e3efc95c6d1fe74ab43d86153e4bf863c09efd36bfe57efc669bc2", + "internal/model/user.go": "4bfd2d78e72f1c80b12541f2abc8500613e8f4d101575bd5d74fac22b52e3617", + "internal/repository/attachment.go": "62e75dbb4ed1cb966abbafc74530a676409ffa0b016a7943be5281cab1dce45c", + "internal/repository/attachment_tx.go": "c24c5b61a8cf07e823835c6f09110ac2334892dc53746b4039688a7250f80e87", + "internal/repository/chat.go": "7ef8dca39ff371f16d351b073199c25f719db22c7a525ab76ea7aca948efa157", + "internal/repository/errors.go": "9e3f77340ff87e41edc87380f795759701980685478737e7e4e1f9737e2634fa", + "internal/repository/message.go": "2f92e2808317850ca5873cc26f8e6c4d83104e8fc957a89304fc74ba2f64cbf2", + "internal/repository/postgres/attachment.go": "5ca7ed710c8dc07473376810e2bf746d2f2d60435b220e2e1cf20ee82c0c3e57", + "internal/repository/postgres/chat.go": "d757c579e5b6599ab9366aef46fd59f9652247f3d947615aa46c919746b65ea5", + "internal/repository/postgres/errors.go": "580e5e599f7af8a403404c6850986e88efceda6393d21e772ac108be34004ba9", + "internal/repository/postgres/message.go": "a2057f6fe6ba2a93a7f98bdbd799baf043999811e9fa7bb649e4f0d60138b6c2", + "internal/repository/postgres/user.go": "eb6770e5cdd96b16a6e9d2116af89a6dd466dfbca4f4c179fe06c30c487e53c4", + "internal/repository/postgres/utils.go": "6c5c83a8ad8ecf611812fbf8c90735e2590a09dec88cd914a10f6b36a9f041b7", + "internal/repository/redis/errors.go": "ab56571d1d0b7d13b7d7ff868916f7e506c5a54c29e6bb211b3021ad4bf601ab", + "internal/repository/redis/session.go": "c1654fea2ba47734a5403d6f6cbb2fe70fbe7e6e37166ae75f338d3490981de1", + "internal/repository/session.go": "071285ff4d8033a6512a7256583a1cb32de782d6fd9f1696fac109927f7db699", + "internal/repository/user.go": "bee65136137d97bea2240411c96a93cd945b1b9145452416f7a83733a1fd28e7", + "internal/security/password.go": "8f74bef9216a2d12aa734e04fa786eac6b31f1378fe42f729fe1b97495c10487", + "internal/service/attachment.go": "4b901d85d93d6dffde1bf76dd532b4b6845afd55ea7a8f9aaea983eaead1e9da", + "internal/service/attachment_cleanup.go": "c4e66bb48f7cd26de6fedf4ca7cf751c815dcab30e9442de61f7fe000d3c2367", + "internal/service/attachment_reconcile.go": "f7dbf60b14f9e82cc018bdd29482d71786a30c32bfb5d1ed1808342c9fd36cdb", + "internal/service/auth.go": "71b8960640866e515e9060159af750c418d97a6373d2616c3b59b764172aa54c", + "internal/service/chat.go": "bc7c8bea0066d8bbd926f78d2c100f0bd9a48c29fb542c5cfa3f2ae415299ef5", + "internal/service/errors.go": "fb5aa4f7be76af14ab81629eceb02e1c3a1bafd8bfeb036e7ee5ccd3940d64da", + "internal/service/helper.go": "390320fa296d9637630e6ac0d90c1229002b578a29186cad414896d5f4d1b8a6", + "internal/service/message.go": "9556a18a5679d7e44b5cdcf329488185d2b90cadb3180af926df4bb6ab3f9d0d", + "internal/service/user.go": "542051ef3eeff15d340979a615a7e0da3d21ed7ed93bea2547ab52d8e303b5e2", + "internal/storage/errors.go": "21a5622cd2655663fc24ee485013c1a917279990fd128a5d406b59f5f0ee67cd", + "internal/storage/object_cleanup_storage.go": "ef50236baefe3e3aea8ca64264ae3c2e5fc89c245b9b1d7cfd207d354f0be2c9", + "internal/storage/object_writer.go": "15ca1257b9c42f64a42f19a4bbb487e9d2950aed2fdf87959fe6c2d763998b64", + "internal/storage/s3/client.go": "032ec339cb91fc1f049a4f92ac61d0404199be059d452307cd12455386f68134", + "internal/storage/s3/errors.go": "6f1f5a7dbd93f65f8101ba80b6fa17bab964744d237746b3ae4ac3650cc9b52c", + "internal/transport/http/handler/attachment.go": "885b2c4d24cbf0f5bfa9def4f6b5b4a7621919b47c3906d89a297abcf0c73da8", + "internal/transport/http/handler/attachment_types.go": "90210075401601bacb25ae70dea2e19a203f537cf0f5aff1369679619bb87811", + "internal/transport/http/handler/auth.go": "f175a570d47ca38ce627d0bc88b6f6a2adbf648345229fa961ea0144717744f1", + "internal/transport/http/handler/auth_types.go": "5ffcd182606a77d538efb5f1a51df23b8a324a605b5d226026048d34c409ebdb", + "internal/transport/http/handler/chat.go": "e5214ebf94ac9b8f37537e1a3fc844a2e8ae4e9b5aa159fe2b6e4da9ab2342f0", + "internal/transport/http/handler/chat_types.go": "d181bce0f003749d813c1254df4c8ebcc1b99ff506b61efa6899da4a499e95f4", + "internal/transport/http/handler/health.go": "1abf04652ea0b69bd87321b95e1a921d3c5ad3c04207232c2a9408bf722e550a", + "internal/transport/http/handler/health_types.go": "02ee3031e10fbb279e020f409fbd905ae2640aa306b2c558c3f8e6dcfe9efa80", + "internal/transport/http/handler/message.go": "540065ffc9090848fc25d560f1ad9734d4f0d45cdf0b5907a0f196f9efa8ede3", + "internal/transport/http/handler/message_types.go": "270a9504a3001b1dc02a1f62cdaed12d69af2471340674e50022c609643dcb34", + "internal/transport/http/handler/user.go": "86fd65ed176e9a83dc27c1112ff8384bc5aa32a92410290fb505d715ca167775", + "internal/transport/http/handler/user_types.go": "0c4450be649afe5edf4e8d3ac9baada79deb9e1423ace56b87aa227ebd0eb998", + "internal/transport/http/httputils/helper.go": "59c092950b69f5390427d0c6c1f90d2c33630d87a42bef363aed704960cb3a22", + "internal/transport/http/middleware/auth.go": "413d7fb5f8573a808c2eb4e36becd29b4d7edffe5b096b26fc07ac22a0285430", + "internal/transport/http/middleware/context.go": "f6db0e807659f30b05cae91f73c70d87f359760c57110e0d25b49a65bd0394a5", + "internal/transport/http/middleware/logging.go": "a477fda512953eb9f46266537c12ace9cc992ba8f036a5b041f9f8295bfd2097", + "internal/transport/http/router.go": "2841bac8988517591539336ba20718b4da7bae31ab81bf56e6b2b9bb59b0a297", + "scripts/restart.sh": "5f70beb4337aed16881a0728229f02dce0e95d42028e0088acd52aa0a76eb019", + "sqlc.yaml": "4e72de8f0b44d6d2b6a44da89f86dd5326e85077a5e7a8ae71816a5dff1fbaa3" + }, + "sha256": "4c7aff2f3780fd4df0ebb4ab7b380e9c8540bbcb45ae64fb0e2894bb650109e5" +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..7a2814f1 --- /dev/null +++ b/go.mod @@ -0,0 +1,43 @@ +module highload + +go 1.27.1 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.11.0 + github.com/pressly/goose/v3 v3.26.0 + github.com/redis/go-redis/v9 v9.23.0 + golang.org/x/crypto v0.57.0 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.48.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 // indirect + github.com/aws/aws-sdk-go-v2/config v1.33.18 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.21.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 // indirect + github.com/aws/smithy-go v1.28.9 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect + github.com/sethvargo/go-retry v0.3.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.42.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..d4989072 --- /dev/null +++ b/go.sum @@ -0,0 +1,105 @@ +github.com/aws/aws-sdk-go-v2 v1.48.0 h1:tFU7EJVm7MfyaXLsXdYTZ0Vd6vwt9OljCyqq0gqk0Mc= +github.com/aws/aws-sdk-go-v2 v1.48.0/go.mod h1:oeKU5p+TQBnpeLmU42WZLWvzv+ORLn+74n4QVLnm/Z0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27 h1:3dMP9eEmHVosFN1QiD24IJBKVY8m6iUfVYXqRCT118Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.27/go.mod h1:KH4PKcqtjy1m0656eqQV2v53xE04ddJzuNloEKqlHkI= +github.com/aws/aws-sdk-go-v2/config v1.33.18 h1:WC0C4jM/tewXG8DWXnoxXnusJPZTryPazqk2k/zgd2s= +github.com/aws/aws-sdk-go-v2/config v1.33.18/go.mod h1:0kRpFgsnD59mC2bbbyOTVpaZ9RnXW/pFkqlfiwbgZaU= +github.com/aws/aws-sdk-go-v2/credentials v1.21.0 h1:OViwhqeipkQzSHuO2eQDUCdLrI6r3nzGxFUyM8GqnCs= +github.com/aws/aws-sdk-go-v2/credentials v1.21.0/go.mod h1:OffaVQod3fyZzhZJyPtw11WpHPF1uIHADc+/MYlXmbA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9 h1:oH1E0tjN0g+ZRusbX1dsuat4FJ0z05l8hWaycSbDuNc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.9/go.mod h1:3ZaQGigIoaS45x9HgnLZ4sP475Fs7QoxU3IsZX/J3h4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12 h1:W0yUUzKBA8SofGQ0yZafEUuZq9UxfvijRgdc6vcRQPA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.12/go.mod h1:xwKv6vwBNdpPFOxEm/ld4IODZK2s/1ksHvdBqLfH6Gs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12 h1:lqSi8qvAnKC1vfosImdmOsuiJLr/jdybZX5299beB6Q= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.12/go.mod h1:rqPQE2vqkW9YHx5+Gp55sx1wsTM4xtGQLl+TKMjxxrs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12 h1:uvX2yaL+aMjkmnrLPPlR/zwjoKFRMoNhhJfeGjfk6qc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.12/go.mod h1:dicT+2eNhHHqGkxHO7LoXw3QghIUGuzgdmFqaSTopgI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25 h1:rKwkY3Z+fvt2W0msjdNs6uNewPmXzPayQ961oyuWhes= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.25/go.mod h1:dBuAadQKd1F4Q4ZOg8gnFmuSHMRE40Pe3IfFwbtC4lE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13 h1:U5HZ8FpTr6tyvulf1nT11Nw4KG1CAsXkuQGdUrIjZ/E= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.13/go.mod h1:dUfvCU1NaE+b3vqNENDiJncUQW7Djl5pmgUlcjesVqk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12 h1:wzAqMZ2O4+ij9HQqIrk+P9PBGacE0wOTuWB7Mo97//I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.12/go.mod h1:/0TfyruByHsOlphzP26xDTRe2DKD0J1I1Uyy2TyP8EI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12 h1:H0IaQayT3TLiGIHiNnleQZTbndDlx7LGZ0NWZ5mDwjA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.12/go.mod h1:YuJJNxaOjnknLIrscBmkLhIpnmg9LzHd25sQRKUboGU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10 h1:WBJb2VxRzzKnuN94YWqZPiu5a5zc3IleFqgZnmVCyvU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.10/go.mod h1:8NhAIBVYkVHq0hIbbzujdoou0VkKwMdG+De9e5oX6qI= +github.com/aws/aws-sdk-go-v2/service/signin v1.13.1 h1:ssrDA6AKfXEc3h0Pf4EI4vQ8sN5Zd136SrU9Tjr0XPc= +github.com/aws/aws-sdk-go-v2/service/signin v1.13.1/go.mod h1:LdwHjKqPE9VCC7sTlWcOAizZx3ZJYjmfwC/EW2tIIAE= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.5 h1:ANAoa+qt3/9kXnvQVgUyxcTzhzZf4DqaBS+M2DQrELU= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.5/go.mod h1:vsMi+SSR7xGQXtV430LVi49qbv/R+U+upHHSp1W/+QI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5 h1:uIOBLMhef4N16WguadoHKuBy9etdz+RKTeqzchXErAI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.5/go.mod h1:NwWmkFLtsTVyXQpbzPOlYLwEoANkdCFjJRtrNcbg/vY= +github.com/aws/aws-sdk-go-v2/service/sts v1.52.0 h1:IaBLmJWiIEtKkjXMgMbYGZk7WKXbYA8148wCMCXq9hU= +github.com/aws/aws-sdk-go-v2/service/sts v1.52.0/go.mod h1:MG6N8U9+lY4LqchrlHPqCOSr7hyxw70XU0Et2YR3eNA= +github.com/aws/smithy-go v1.28.9 h1:zkZ058OpNX0iGTK24b+ZUEQSMc11CBA6oO7xtKZfzkY= +github.com/aws/smithy-go v1.28.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.26.0 h1:KJakav68jdH0WDvoAcj8+n61WqOIaPGgH0bJWS6jpmM= +github.com/pressly/goose/v3 v3.26.0/go.mod h1:4hC1KrritdCxtuFsqgs1R4AU5bWtTAf+cnWvfhf2DNY= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= +github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= +golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= +modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= +modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..17bf0a36 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,28 @@ +global + log stdout format raw local0 + stats socket ipv4@127.0.0.1:9999 level admin + +defaults + mode http + log global + option httplog + timeout connect 2s + timeout client 30s + timeout server 30s + timeout http-request 10s + +resolvers docker + nameserver docker 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + +frontend api + bind *:8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + timeout check 3s + option httpchk GET /health/ready + http-check expect status 200 + server-template api 4 api:8080 check inter 1s fall 2 rise 2 resolvers docker init-addr none diff --git a/infra/s3/bootstrap.sh b/infra/s3/bootstrap.sh new file mode 100755 index 00000000..0a888d25 --- /dev/null +++ b/infra/s3/bootstrap.sh @@ -0,0 +1,65 @@ +#!/bin/sh +set -eu + +for name in rustfs_root_access_key rustfs_root_secret_key s3_access_key_id s3_secret_access_key; do + if [ ! -s "/run/secrets/$name" ]; then + echo "missing S3 secret: $name" >&2 + exit 1 + fi +done + +root_key=$(cat /run/secrets/rustfs_root_access_key) +root_secret=$(cat /run/secrets/rustfs_root_secret_key) +app_key=$(cat /run/secrets/s3_access_key_id) +app_secret=$(cat /run/secrets/s3_secret_access_key) + +if [ "$root_key" = "$app_key" ]; then + echo 'API and RustFS root access keys must differ' >&2 + exit 1 +fi + +# These values become path and JSON fragments below. +case "$S3_BUCKET" in + messenger-attachments) ;; + *) echo 'unsupported S3_BUCKET in bootstrap' >&2; exit 1 ;; +esac + +rc=/usr/bin/rc +policy=messenger-attachments-api-v1 + +"$rc" alias set root http://s3:9000 "$root_key" "$root_secret" \ + --region "$S3_REGION" --bucket-lookup path >/dev/null +"$rc" bucket create "root/$S3_BUCKET" --ignore-existing +# A new bucket is private by default; this also removes a stale public policy. +"$rc" bucket anonymous set private "root/$S3_BUCKET" + +cat > /tmp/api-policy.json </dev/null 2>&1; then + "$rc" admin policy create root/ "$policy" /tmp/api-policy.json +fi +if ! "$rc" admin user info root/ "$app_key" >/dev/null 2>&1; then + "$rc" admin user add root/ "$app_key" "$app_secret" >/dev/null +fi +"$rc" admin policy attach root/ "$policy" --user "$app_key" >/dev/null + +"$rc" alias set app http://s3:9000 "$app_key" "$app_secret" \ + --region "$S3_REGION" --bucket-lookup path >/dev/null +"$rc" bucket list "app/$S3_BUCKET" >/dev/null +echo "S3 bucket and API identity ready: $S3_BUCKET" diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..2760d9a8 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,175 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type Database struct { + PostgresURL string + RedisAddr string +} + +type S3 struct { + Endpoint string + Region string + Bucket string + AccessKeyID string + SecretAccessKey string +} + +type Config struct { + HTTP struct { + Host string + Port string + } + Database Database + + Session struct { + TTL time.Duration + Store string + } + + S3 S3 + + AppEnv string + CacheURL string +} + +func Load() (*Config, error) { + config := &Config{} + sessionTTL, err := getSessionTTL() + + if err != nil { + return nil, err + } + config.Session.TTL = sessionTTL + + config.HTTP.Host = os.Getenv("APP_HOST") + if config.HTTP.Host == "" { + config.HTTP.Host = "0.0.0.0" + } + + config.HTTP.Port = os.Getenv("APP_PORT") + if config.HTTP.Port == "" { + config.HTTP.Port = "8080" + } + + config.AppEnv = os.Getenv("APP_ENV") + if config.AppEnv == "" { + config.AppEnv = "lab" + } + + store := os.Getenv("SESSION_STORE") + + config.Session.Store = strings.TrimSpace(string(store)) + if config.Session.Store == "" { + return nil, fmt.Errorf("empty session store data") + } + + config.Database, err = getDatabaseURL() + if err != nil { + return nil, err + } + + config.S3, err = getS3Data() + + if err != nil { + return nil, err + } + + return config, nil + +} + +func getSessionTTL() (time.Duration, error) { + value, ok := os.LookupEnv("SESSION_TTL_SECONDS") + if !ok { + return time.Hour, nil + } + + seconds, err := strconv.ParseInt(value, 10, 64) + if err != nil || seconds <= 0 || seconds > int64((1<<63-1)/int64(time.Second)) { + return 0, fmt.Errorf("invalid SESSION_TTL_SECONDS: %q", value) + } + + return time.Duration(seconds) * time.Second, nil +} + +func LoadAttachmentCleanup() (postgresURL string, objectStorage S3, err error) { + postgresURL, err = loadPostgresURL() + if err != nil { + return "", S3{}, err + } + objectStorage, err = getS3Data() + return postgresURL, objectStorage, err +} + +func loadPostgresURL() (string, error) { + postgresData, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE")) + if err != nil { + return "", fmt.Errorf("read database URL: %w", err) + } + postgresURL := strings.TrimSpace(string(postgresData)) + if postgresURL == "" { + return "", fmt.Errorf("empty database URL") + } + return postgresURL, nil +} + +func getDatabaseURL() (Database, error) { + postgresURL, err := loadPostgresURL() + if err != nil { + return Database{}, err + } + db := Database{PostgresURL: postgresURL} + + redisData := os.Getenv("REDIS_ADDR") + + db.RedisAddr = strings.TrimSpace(string(redisData)) + + if db.RedisAddr == "" { + return db, fmt.Errorf("redis addres is empty") + } + + return db, nil + +} + +func getS3Data() (S3, error) { + s3 := S3{ + Endpoint: os.Getenv("S3_ENDPOINT"), + Region: os.Getenv("S3_REGION"), + Bucket: os.Getenv("S3_BUCKET"), + } + + if s3.Endpoint == "" { + return s3, fmt.Errorf("empty s3 endpoint") + } + if s3.Region == "" { + return s3, fmt.Errorf("empty s3 region") + } + if s3.Bucket == "" { + return s3, fmt.Errorf("emtpy s3 bucket") + } + accessKeyIDPath := os.Getenv("S3_ACCESS_KEY_ID_FILE") + + accessKeyID, err := os.ReadFile(accessKeyIDPath) + if err != nil { + return s3, fmt.Errorf("empty s3 access_key_id") + } + s3.AccessKeyID = strings.TrimSpace(string(accessKeyID)) + + secretAccessKeyPath := os.Getenv("S3_SECRET_ACCESS_KEY_FILE") + secretAccessKey, err := os.ReadFile(secretAccessKeyPath) + if err != nil { + return s3, fmt.Errorf("empty s3 secret_access_key_file") + } + s3.SecretAccessKey = strings.TrimSpace(string(secretAccessKey)) + + return s3, nil + +} diff --git a/internal/database/postgres.go b/internal/database/postgres.go new file mode 100644 index 00000000..76a8d2f9 --- /dev/null +++ b/internal/database/postgres.go @@ -0,0 +1,21 @@ +package database + +import ( + "context" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func NewPostgres(ctx context.Context, databaseURL string) (*pgxpool.Pool, error) { + pool, err := pgxpool.New(ctx, databaseURL) + if err != nil { + return nil, err + } + + if err = pool.Ping(ctx); err != nil { + pool.Close() + return nil, err + } + + return pool, nil +} diff --git a/internal/database/redis.go b/internal/database/redis.go new file mode 100644 index 00000000..5ebefaa2 --- /dev/null +++ b/internal/database/redis.go @@ -0,0 +1,31 @@ +package database + +import ( + "context" + "time" + + "github.com/redis/go-redis/v9" +) + +func NewRedis(ctx context.Context, addr string) (*redis.Client, error) { + + options := &redis.Options{ + Addr: addr, + DialTimeout: 3 * time.Second, + ReadTimeout: 3 * time.Second, + WriteTimeout: 2 * time.Second, + PoolSize: 10, + PoolTimeout: 2 * time.Second, + ContextTimeoutEnabled: true, + } + + client := redis.NewClient(options) + + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, err + } + + return client, nil + +} diff --git a/internal/database/sqlc/attachments.sql.go b/internal/database/sqlc/attachments.sql.go new file mode 100644 index 00000000..e43d43a4 --- /dev/null +++ b/internal/database/sqlc/attachments.sql.go @@ -0,0 +1,207 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: attachments.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const attachmentExistsByObjectKey = `-- name: AttachmentExistsByObjectKey :one +SELECT EXISTS ( + SELECT 1 FROM attachments + WHERE object_key = $1 +) +` + +func (q *Queries) AttachmentExistsByObjectKey(ctx context.Context, objectKey string) (bool, error) { + row := q.db.QueryRow(ctx, attachmentExistsByObjectKey, objectKey) + var exists bool + err := row.Scan(&exists) + return exists, err +} + +const createAttachment = `-- name: CreateAttachment :one +INSERT INTO attachments + (chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) +VALUES ($1, $2, $3, $4, $5, $6, $7) +RETURNING id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at +` + +type CreateAttachmentParams struct { + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ObjectKey string + ContentType string + SizeBytes int64 + Sha256 string +} + +func (q *Queries) CreateAttachment(ctx context.Context, arg CreateAttachmentParams) (Attachment, error) { + row := q.db.QueryRow(ctx, createAttachment, + arg.ChatID, + arg.OwnerID, + arg.Filename, + arg.ObjectKey, + arg.ContentType, + arg.SizeBytes, + arg.Sha256, + ) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getAttachmentByID = `-- name: GetAttachmentByID :one +SELECT id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256, created_at, updated_at FROM attachments WHERE id = $1 +` + +func (q *Queries) GetAttachmentByID(ctx context.Context, attachmentID pgtype.UUID) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachmentByID, attachmentID) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getAttachmentForUser = `-- name: GetAttachmentForUser :one +SELECT a.id, a.chat_id, a.owner_id, a.filename, a.object_key, a.content_type, a.size_bytes, a.sha256, a.created_at, a.updated_at +FROM attachments AS a +JOIN chat_members AS cm ON cm.chat_id = a.chat_id +WHERE a.id = $1 + AND cm.user_id = $2 +` + +type GetAttachmentForUserParams struct { + AttachmentID pgtype.UUID + UserID pgtype.UUID +} + +func (q *Queries) GetAttachmentForUser(ctx context.Context, arg GetAttachmentForUserParams) (Attachment, error) { + row := q.db.QueryRow(ctx, getAttachmentForUser, arg.AttachmentID, arg.UserID) + var i Attachment + err := row.Scan( + &i.ID, + &i.ChatID, + &i.OwnerID, + &i.Filename, + &i.ObjectKey, + &i.ContentType, + &i.SizeBytes, + &i.Sha256, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const getAttachmentsForMessages = `-- name: GetAttachmentsForMessages :many +SELECT message_id, attachment_id, position +FROM messages_attachments +WHERE message_id = ANY($1::uuid[]) +ORDER BY message_id, position +` + +type GetAttachmentsForMessagesRow struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + Position int16 +} + +func (q *Queries) GetAttachmentsForMessages(ctx context.Context, messageIds []pgtype.UUID) ([]GetAttachmentsForMessagesRow, error) { + rows, err := q.db.Query(ctx, getAttachmentsForMessages, messageIds) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetAttachmentsForMessagesRow + for rows.Next() { + var i GetAttachmentsForMessagesRow + if err := rows.Scan(&i.MessageID, &i.AttachmentID, &i.Position); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const listAttachmentKeys = `-- name: ListAttachmentKeys :many +SELECT id, object_key, created_at +FROM attachments +WHERE object_key > $1 +ORDER BY object_key +LIMIT $2 +` + +type ListAttachmentKeysParams struct { + AfterKey string + PageSize int32 +} + +type ListAttachmentKeysRow struct { + ID pgtype.UUID + ObjectKey string + CreatedAt pgtype.Timestamptz +} + +func (q *Queries) ListAttachmentKeys(ctx context.Context, arg ListAttachmentKeysParams) ([]ListAttachmentKeysRow, error) { + rows, err := q.db.Query(ctx, listAttachmentKeys, arg.AfterKey, arg.PageSize) + if err != nil { + return nil, err + } + defer rows.Close() + var items []ListAttachmentKeysRow + for rows.Next() { + var i ListAttachmentKeysRow + if err := rows.Scan(&i.ID, &i.ObjectKey, &i.CreatedAt); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const tryLockAttachmentObjectKey = `-- name: TryLockAttachmentObjectKey :one +SELECT pg_try_advisory_xact_lock( + hashtextextended($1::text, 0) +)::boolean AS locked +` + +func (q *Queries) TryLockAttachmentObjectKey(ctx context.Context, objectKey string) (bool, error) { + row := q.db.QueryRow(ctx, tryLockAttachmentObjectKey, objectKey) + var locked bool + err := row.Scan(&locked) + return locked, err +} diff --git a/internal/database/sqlc/chat_members.sql.go b/internal/database/sqlc/chat_members.sql.go new file mode 100644 index 00000000..10326c9c --- /dev/null +++ b/internal/database/sqlc/chat_members.sql.go @@ -0,0 +1,27 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: chat_members.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createChatMember = `-- name: CreateChatMember :exec +INSERT INTO chat_members(chat_id, user_id) +VALUES ($1, $2) +` + +type CreateChatMemberParams struct { + ChatID pgtype.UUID + UserID pgtype.UUID +} + +func (q *Queries) CreateChatMember(ctx context.Context, arg CreateChatMemberParams) error { + _, err := q.db.Exec(ctx, createChatMember, arg.ChatID, arg.UserID) + return err +} diff --git a/internal/database/sqlc/chats.sql.go b/internal/database/sqlc/chats.sql.go new file mode 100644 index 00000000..ce96287a --- /dev/null +++ b/internal/database/sqlc/chats.sql.go @@ -0,0 +1,118 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: chats.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createChat = `-- name: CreateChat :one +INSERT INTO chats (title) +VALUES ($1) +RETURNING id, title, created_at +` + +func (q *Queries) CreateChat(ctx context.Context, title string) (Chat, error) { + row := q.db.QueryRow(ctx, createChat, title) + var i Chat + err := row.Scan(&i.ID, &i.Title, &i.CreatedAt) + return i, err +} + +const getChatByID = `-- name: GetChatByID :one + +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE c.id = $1 +GROUP BY c.id +` + +type GetChatByIDRow struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz + MemberIds []pgtype.UUID +} + +func (q *Queries) GetChatByID(ctx context.Context, id pgtype.UUID) (GetChatByIDRow, error) { + row := q.db.QueryRow(ctx, getChatByID, id) + var i GetChatByIDRow + err := row.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.MemberIds, + ) + return i, err +} + +const getChatsByUserID = `-- name: GetChatsByUserID :many +SELECT c.id, c.title, c.created_at, array_agg(cm.user_id)::uuid[] AS member_ids +FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id +WHERE EXISTS (SELECT 1 + FROM chat_members membership + WHERE membership.chat_id = c.id + AND membership.user_id = $1) + AND ( + $2::timestamptz IS NULL + OR (c.created_at, c.id) > + ( + $2::timestamptz, + $3::uuid + ) + ) +GROUP BY c.id +ORDER BY c.created_at ASC, c.id ASC +LIMIT $4 +` + +type GetChatsByUserIDParams struct { + UserID pgtype.UUID + AfterCreatedAt pgtype.Timestamptz + AfterID pgtype.UUID + LimitCount int32 +} + +type GetChatsByUserIDRow struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz + MemberIds []pgtype.UUID +} + +func (q *Queries) GetChatsByUserID(ctx context.Context, arg GetChatsByUserIDParams) ([]GetChatsByUserIDRow, error) { + rows, err := q.db.Query(ctx, getChatsByUserID, + arg.UserID, + arg.AfterCreatedAt, + arg.AfterID, + arg.LimitCount, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetChatsByUserIDRow + for rows.Next() { + var i GetChatsByUserIDRow + if err := rows.Scan( + &i.ID, + &i.Title, + &i.CreatedAt, + &i.MemberIds, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/db.go b/internal/database/sqlc/db.go new file mode 100644 index 00000000..a28f6fc5 --- /dev/null +++ b/internal/database/sqlc/db.go @@ -0,0 +1,32 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +type DBTX interface { + Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) + Query(context.Context, string, ...interface{}) (pgx.Rows, error) + QueryRow(context.Context, string, ...interface{}) pgx.Row +} + +func New(db DBTX) *Queries { + return &Queries{db: db} +} + +type Queries struct { + db DBTX +} + +func (q *Queries) WithTx(tx pgx.Tx) *Queries { + return &Queries{ + db: tx, + } +} diff --git a/internal/database/sqlc/message_idempotence.sql.go b/internal/database/sqlc/message_idempotence.sql.go new file mode 100644 index 00000000..05052880 --- /dev/null +++ b/internal/database/sqlc/message_idempotence.sql.go @@ -0,0 +1,66 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: message_idempotence.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const getIdempotentMessage = `-- name: GetIdempotentMessage :one +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at +FROM message_idempotency AS i +JOIN messages AS m ON m.id = i.message_id +WHERE i.user_id = $1 + AND i.chat_id = $2 + AND i.idempotency_key = $3 +` + +type GetIdempotentMessageParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string +} + +func (q *Queries) GetIdempotentMessage(ctx context.Context, arg GetIdempotentMessageParams) (Message, error) { + row := q.db.QueryRow(ctx, getIdempotentMessage, arg.UserID, arg.ChatID, arg.IdempotencyKey) + var i Message + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ) + return i, err +} + +const insertMessageIdempotency = `-- name: InsertMessageIdempotency :execrows +INSERT INTO message_idempotency(user_id, chat_id, idempotency_key, message_id) +VALUES ($1, $2, $3, $4) +ON CONFLICT (user_id, chat_id, idempotency_key) DO NOTHING +` + +type InsertMessageIdempotencyParams struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string + MessageID pgtype.UUID +} + +func (q *Queries) InsertMessageIdempotency(ctx context.Context, arg InsertMessageIdempotencyParams) (int64, error) { + result, err := q.db.Exec(ctx, insertMessageIdempotency, + arg.UserID, + arg.ChatID, + arg.IdempotencyKey, + arg.MessageID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/database/sqlc/messages.sql.go b/internal/database/sqlc/messages.sql.go new file mode 100644 index 00000000..aa11ba59 --- /dev/null +++ b/internal/database/sqlc/messages.sql.go @@ -0,0 +1,91 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: messages.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createMessage = `-- name: CreateMessage :one +INSERT INTO messages (chat_id, sender_id, text) +VALUES ($1, $2, $3) +RETURNING id, chat_id, sender_id, text, created_at +` + +type CreateMessageParams struct { + ChatID pgtype.UUID + SenderID pgtype.UUID + Text string +} + +func (q *Queries) CreateMessage(ctx context.Context, arg CreateMessageParams) (Message, error) { + row := q.db.QueryRow(ctx, createMessage, arg.ChatID, arg.SenderID, arg.Text) + var i Message + err := row.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ) + return i, err +} + +const getMessagesByChatID = `-- name: GetMessagesByChatID :many + +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = $1 + AND ( + $2::timestamptz IS NULL + OR (created_at, id) > + ($2::timestamptz, + $3::uuid + ) + ) +ORDER BY created_at ASC, id ASC +LIMIT $4 +` + +type GetMessagesByChatIDParams struct { + ChatID pgtype.UUID + AfterCreatedAt pgtype.Timestamptz + AfterID pgtype.UUID + LimitCount int32 +} + +func (q *Queries) GetMessagesByChatID(ctx context.Context, arg GetMessagesByChatIDParams) ([]Message, error) { + rows, err := q.db.Query(ctx, getMessagesByChatID, + arg.ChatID, + arg.AfterCreatedAt, + arg.AfterID, + arg.LimitCount, + ) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Message + for rows.Next() { + var i Message + if err := rows.Scan( + &i.ID, + &i.ChatID, + &i.SenderID, + &i.Text, + &i.CreatedAt, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/messages_attachments.sql.go b/internal/database/sqlc/messages_attachments.sql.go new file mode 100644 index 00000000..df39301b --- /dev/null +++ b/internal/database/sqlc/messages_attachments.sql.go @@ -0,0 +1,71 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: messages_attachments.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createMessagesAttachments = `-- name: CreateMessagesAttachments :execrows +INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) +SELECT $1, $2, $3, $4 +FROM attachments +WHERE id = $2 AND chat_id = $3 +` + +type CreateMessagesAttachmentsParams struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + ChatID pgtype.UUID + Position int16 +} + +func (q *Queries) CreateMessagesAttachments(ctx context.Context, arg CreateMessagesAttachmentsParams) (int64, error) { + result, err := q.db.Exec(ctx, createMessagesAttachments, + arg.MessageID, + arg.AttachmentID, + arg.ChatID, + arg.Position, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getMessagesAttachments = `-- name: GetMessagesAttachments :many +SELECT attachment_id, position +FROM messages_attachments +WHERE message_id = $1 +ORDER BY position ASC +` + +type GetMessagesAttachmentsRow struct { + AttachmentID pgtype.UUID + Position int16 +} + +func (q *Queries) GetMessagesAttachments(ctx context.Context, messageID pgtype.UUID) ([]GetMessagesAttachmentsRow, error) { + rows, err := q.db.Query(ctx, getMessagesAttachments, messageID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []GetMessagesAttachmentsRow + for rows.Next() { + var i GetMessagesAttachmentsRow + if err := rows.Scan(&i.AttachmentID, &i.Position); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/database/sqlc/models.go b/internal/database/sqlc/models.go new file mode 100644 index 00000000..2581238b --- /dev/null +++ b/internal/database/sqlc/models.go @@ -0,0 +1,65 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 + +package sqlc + +import ( + "github.com/jackc/pgx/v5/pgtype" +) + +type Attachment struct { + ID pgtype.UUID + ChatID pgtype.UUID + OwnerID pgtype.UUID + Filename string + ObjectKey string + ContentType string + SizeBytes int64 + Sha256 string + CreatedAt pgtype.Timestamptz + UpdatedAt pgtype.Timestamptz +} + +type Chat struct { + ID pgtype.UUID + Title string + CreatedAt pgtype.Timestamptz +} + +type ChatMember struct { + ChatID pgtype.UUID + UserID pgtype.UUID +} + +type Message struct { + ID pgtype.UUID + ChatID pgtype.UUID + SenderID pgtype.UUID + Text string + CreatedAt pgtype.Timestamptz +} + +type MessageIdempotency struct { + UserID pgtype.UUID + ChatID pgtype.UUID + IdempotencyKey string + MessageID pgtype.UUID + CreatedAt pgtype.Timestamptz + ExpiresAt pgtype.Timestamptz +} + +type MessagesAttachment struct { + MessageID pgtype.UUID + AttachmentID pgtype.UUID + ChatID pgtype.UUID + Position int16 +} + +type User struct { + ID pgtype.UUID + Username string + DisplayName string + PasswordHash string + CreatedAt pgtype.Timestamptz +} diff --git a/internal/database/sqlc/users.sql.go b/internal/database/sqlc/users.sql.go new file mode 100644 index 00000000..6e423e55 --- /dev/null +++ b/internal/database/sqlc/users.sql.go @@ -0,0 +1,77 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.31.1 +// source: users.sql + +package sqlc + +import ( + "context" + + "github.com/jackc/pgx/v5/pgtype" +) + +const createUser = `-- name: CreateUser :one + +INSERT INTO users (username, display_name, password_hash) +VALUES ($1, $2, $3) +RETURNING id, username, display_name, password_hash, created_at +` + +type CreateUserParams struct { + Username string + DisplayName string + PasswordHash string +} + +func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (User, error) { + row := q.db.QueryRow(ctx, createUser, arg.Username, arg.DisplayName, arg.PasswordHash) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} + +const getUserByID = `-- name: GetUserByID :one + +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE id = $1 +` + +func (q *Queries) GetUserByID(ctx context.Context, id pgtype.UUID) (User, error) { + row := q.db.QueryRow(ctx, getUserByID, id) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} + +const getUserByUsername = `-- name: GetUserByUsername :one +SELECT id, username, display_name, password_hash, created_at +FROM users +WHERE username = $1 +` + +func (q *Queries) GetUserByUsername(ctx context.Context, username string) (User, error) { + row := q.db.QueryRow(ctx, getUserByUsername, username) + var i User + err := row.Scan( + &i.ID, + &i.Username, + &i.DisplayName, + &i.PasswordHash, + &i.CreatedAt, + ) + return i, err +} diff --git a/internal/model/attachment.go b/internal/model/attachment.go new file mode 100644 index 00000000..51561bc8 --- /dev/null +++ b/internal/model/attachment.go @@ -0,0 +1,16 @@ +package model + +import "time" + +type Attachment struct { + ID string + ChatID string + OwnerID string + Filename string + ObjectKey string + ContentType string + SizeBytes int64 + SHA256 string + CreatedAt time.Time + UpdatedAt time.Time +} diff --git a/internal/model/chat.go b/internal/model/chat.go new file mode 100644 index 00000000..2e58c626 --- /dev/null +++ b/internal/model/chat.go @@ -0,0 +1,10 @@ +package model + +import "time" + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} diff --git a/internal/model/message.go b/internal/model/message.go new file mode 100644 index 00000000..7e969c88 --- /dev/null +++ b/internal/model/message.go @@ -0,0 +1,12 @@ +package model + +import "time" + +type Message struct { + ID string + ChatID string + SenderID string + Text string + AttachmentIDs []string + CreatedAt time.Time +} diff --git a/internal/model/session.go b/internal/model/session.go new file mode 100644 index 00000000..aa790b76 --- /dev/null +++ b/internal/model/session.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time + RevokedAt *time.Time +} diff --git a/internal/model/user.go b/internal/model/user.go new file mode 100644 index 00000000..f2209764 --- /dev/null +++ b/internal/model/user.go @@ -0,0 +1,11 @@ +package model + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} diff --git a/internal/repository/attachment.go b/internal/repository/attachment.go new file mode 100644 index 00000000..1fbcedea --- /dev/null +++ b/internal/repository/attachment.go @@ -0,0 +1,36 @@ +package repository + +import ( + "context" + "highload/internal/model" + "time" +) + +type AttachmentRepository interface { + CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error) + GetAttachmentForUser(ctx context.Context, userID, attachmentID string) (*model.Attachment, error) + ExistsByObjectKey(ctx context.Context, key string) (bool, error) + WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error) +} + +type AttachmentKey struct { + ID string + ObjectKey string + CreatedAt time.Time +} + +type AttachmentReconcileRepository interface { + ListAttachmentKeys(ctx context.Context, afterKey string, pageSize int32) ([]AttachmentKey, error) + GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) + WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, AttachmentTx) error) (bool, error) +} + +type CreateAttachmentParams struct { + ChatID string + OwnerID string + Filename string + ObjectKey string + ContentType string + Sha256 string + SizeBytes int64 +} diff --git a/internal/repository/attachment_tx.go b/internal/repository/attachment_tx.go new file mode 100644 index 00000000..844ed9ae --- /dev/null +++ b/internal/repository/attachment_tx.go @@ -0,0 +1,12 @@ +package repository + +import ( + "context" + "highload/internal/model" +) + +type AttachmentTx interface { + CreateAttachment(ctx context.Context, params CreateAttachmentParams) (*model.Attachment, error) + ExistsByObjectKey(ctx context.Context, key string) (bool, error) + GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) +} diff --git a/internal/repository/chat.go b/internal/repository/chat.go new file mode 100644 index 00000000..5c1bce7f --- /dev/null +++ b/internal/repository/chat.go @@ -0,0 +1,18 @@ +package repository + +import ( + "context" + "highload/internal/model" + "time" +) + +type ChatRepository interface { + CreateChat(ctx context.Context, title string, memberIDs []string) (*model.Chat, error) + GetChatByID(ctx context.Context, id string) (*model.Chat, error) + GetChatsByUserID(ctx context.Context, userID string, limit int, after *ChatPosition) ([]*model.Chat, error) +} + +type ChatPosition struct { + CreatedAt time.Time + ID string +} diff --git a/internal/repository/errors.go b/internal/repository/errors.go new file mode 100644 index 00000000..23b4bad0 --- /dev/null +++ b/internal/repository/errors.go @@ -0,0 +1,16 @@ +package repository + +import ( + "errors" +) + +var ( + ErrNotFound = errors.New("not found") + ErrAlreadyExists = errors.New("already exists") + ErrDatabase = errors.New("database error") + ErrInvalidInput = errors.New("invalid input") + ErrServiceUnavailable = errors.New("database unavailable") + ErrServerError = errors.New("server error") + ErrRedisNotAnswer = errors.New("redis not not answer") + ErrConflict = errors.New("data conflict") +) diff --git a/internal/repository/message.go b/internal/repository/message.go new file mode 100644 index 00000000..f4c3c613 --- /dev/null +++ b/internal/repository/message.go @@ -0,0 +1,17 @@ +package repository + +import ( + "context" + "highload/internal/model" + "time" +) + +type MessageRepository interface { + CreateMessage(ctx context.Context, chatID, senderID, text, key string, attachmentIDs []string) (*model.Message, error) + GetMessagesByChatID(ctx context.Context, chatID string, limit int, after *MessagePosition) ([]*model.Message, error) +} + +type MessagePosition struct { + CreatedAt time.Time + ID string +} diff --git a/internal/repository/postgres/attachment.go b/internal/repository/postgres/attachment.go new file mode 100644 index 00000000..a64fdf07 --- /dev/null +++ b/internal/repository/postgres/attachment.go @@ -0,0 +1,188 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "highload/internal/repository" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const attachmentTransactionTimeout = 5 * time.Second + +type AttachmentRepository struct { + queries *sqlc.Queries + pool *pgxpool.Pool +} + +func NewAttachmentRepository(queries *sqlc.Queries, pool *pgxpool.Pool) *AttachmentRepository { + return &AttachmentRepository{ + queries: queries, + pool: pool, + } +} + +func (r *AttachmentRepository) CreateAttachment(ctx context.Context, params repository.CreateAttachmentParams) (*model.Attachment, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + chatUUID, err := getUUIDFromString(params.ChatID) + if err != nil { + return nil, err + } + ownerUUID, err := getUUIDFromString(params.OwnerID) + if err != nil { + return nil, err + } + + sqlcParams := sqlc.CreateAttachmentParams{ + ChatID: *chatUUID, + OwnerID: *ownerUUID, + Filename: params.Filename, + ObjectKey: params.ObjectKey, + ContentType: params.ContentType, + SizeBytes: params.SizeBytes, + Sha256: params.Sha256, + } + + sqlcAttachment, err := r.queries.CreateAttachment(dbCtx, sqlcParams) + + if err != nil { + return nil, convertDBError(err) + } + + attachment := mapAttachmentSqlcToModel(sqlcAttachment) + return attachment, nil + +} + +func (r *AttachmentRepository) GetAttachmentForUser(ctx context.Context, userID, attachmentID string) (*model.Attachment, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + userUUID, err := getUUIDFromString(userID) + if err != nil { + return nil, err + } + attachmentUUID, err := getUUIDFromString(attachmentID) + if err != nil { + return nil, err + } + + params := sqlc.GetAttachmentForUserParams{ + AttachmentID: *attachmentUUID, + UserID: *userUUID, + } + + sqlcAttachment, err := r.queries.GetAttachmentForUser(dbCtx, params) + if err != nil { + return nil, convertDBError(err) + } + + attachment := mapAttachmentSqlcToModel(sqlcAttachment) + return attachment, nil + +} + +func (r *AttachmentRepository) ExistsByObjectKey(ctx context.Context, key string) (bool, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + exists, err := r.queries.AttachmentExistsByObjectKey(dbCtx, key) + if err != nil { + return false, convertDBError(err) + } + return exists, nil +} + +func (r *AttachmentRepository) ListAttachmentKeys(ctx context.Context, afterKey string, pageSize int32) ([]repository.AttachmentKey, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + rows, err := r.queries.ListAttachmentKeys(dbCtx, sqlc.ListAttachmentKeysParams{AfterKey: afterKey, PageSize: pageSize}) + if err != nil { + return nil, convertDBError(err) + } + keys := make([]repository.AttachmentKey, 0, len(rows)) + for _, row := range rows { + keys = append(keys, repository.AttachmentKey{ + ID: row.ID.String(), ObjectKey: row.ObjectKey, CreatedAt: row.CreatedAt.Time, + }) + } + return keys, nil +} + +func (r *AttachmentRepository) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + attachmentID, err := getUUIDFromString(id) + if err != nil { + return nil, err + } + row, err := r.queries.GetAttachmentByID(dbCtx, *attachmentID) + if err != nil { + return nil, convertDBError(err) + } + return mapAttachmentSqlcToModel(row), nil +} + +func (r *AttachmentRepository) WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, repository.AttachmentTx) error) (bool, error) { + txCtx, cancel := context.WithTimeout(ctx, attachmentTransactionTimeout) + defer cancel() + + tx, err := r.pool.BeginTx(txCtx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted}) + if err != nil { + return false, convertDBError(err) + } + + defer func() { + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + time.Second, + ) + defer rollbackCancel() + + _ = tx.Rollback(rollbackCtx) + }() + txQueries := r.queries.WithTx(tx) + + locked, err := txQueries.TryLockAttachmentObjectKey(txCtx, key) + + if err != nil { + return false, convertDBError(err) + } + + if !locked { + return false, nil + } + + txRepo := &AttachmentRepository{queries: txQueries} + + if err = fn(txCtx, txRepo); err != nil { + return true, err + } + + if err = tx.Commit(txCtx); err != nil { + return true, convertDBError(err) + } + + return true, nil +} + +func mapAttachmentSqlcToModel(sa sqlc.Attachment) *model.Attachment { + return &model.Attachment{ + ID: sa.ID.String(), + ChatID: sa.ChatID.String(), + OwnerID: sa.OwnerID.String(), + Filename: sa.Filename, + ObjectKey: sa.ObjectKey, + ContentType: sa.ContentType, + SizeBytes: sa.SizeBytes, + SHA256: sa.Sha256, + CreatedAt: sa.CreatedAt.Time, + UpdatedAt: sa.UpdatedAt.Time, + } + +} diff --git a/internal/repository/postgres/chat.go b/internal/repository/postgres/chat.go new file mode 100644 index 00000000..aedb3fb1 --- /dev/null +++ b/internal/repository/postgres/chat.go @@ -0,0 +1,163 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "highload/internal/repository" + "slices" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +type ChatRepository struct { + queries *sqlc.Queries + pool *pgxpool.Pool +} + +func NewChatRepository(queries *sqlc.Queries, pool *pgxpool.Pool) *ChatRepository { + return &ChatRepository{ + queries: queries, + pool: pool, + } +} + +func (r *ChatRepository) CreateChat(ctx context.Context, title string, memberIDs []string) (*model.Chat, error) { + txCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + tx, err := r.pool.Begin(txCtx) + if err != nil { + return nil, convertDBError(err) + } + + defer func() { + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + time.Second, + ) + defer rollbackCancel() + + _ = tx.Rollback(rollbackCtx) + }() + txQueries := r.queries.WithTx(tx) + + sqlcChat, err := txQueries.CreateChat(txCtx, title) + if err != nil { + return nil, convertDBError(err) + } + + chat := &model.Chat{ + ID: sqlcChat.ID.String(), + Title: sqlcChat.Title, + CreatedAt: sqlcChat.CreatedAt.Time, + } + + for _, id := range memberIDs { + userID, err := getUUIDFromString(id) + if err != nil { + return nil, convertDBError(err) + } + err = txQueries.CreateChatMember(txCtx, sqlc.CreateChatMemberParams{ChatID: sqlcChat.ID, UserID: *userID}) + + if err != nil { + return nil, convertDBError(err) + } + } + + chat.MemberIDs = slices.Clone(memberIDs) + + err = tx.Commit(txCtx) + if err != nil { + return nil, convertDBError(err) + } + + return chat, nil + +} + +func (r *ChatRepository) GetChatByID(ctx context.Context, id string) (*model.Chat, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidID, err := getUUIDFromString(id) + + if err != nil { + return nil, convertDBError(err) + } + + sqlcChat, err := r.queries.GetChatByID(dbCtx, *uuidID) + + if err != nil { + return nil, convertDBError(err) + } + + chat := mapChatSqlcToModel(&sqlcChat) + + return chat, nil + +} + +func (r *ChatRepository) GetChatsByUserID(ctx context.Context, userID string, limit int, after *repository.ChatPosition) ([]*model.Chat, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidUserID, err := getUUIDFromString(userID) + + if err != nil { + return nil, convertDBError(err) + } + + afterCreatedAt := pgtype.Timestamptz{Valid: false} + afterID := pgtype.UUID{Valid: false} + + if after != nil { + afterCreatedAt = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true} + if uuidFromString, err := getUUIDFromString(after.ID); err == nil { + afterID = *uuidFromString + } else { + return nil, convertDBError(err) + } + } + + chats, err := r.queries.GetChatsByUserID(dbCtx, sqlc.GetChatsByUserIDParams{UserID: *uuidUserID, AfterCreatedAt: afterCreatedAt, AfterID: afterID, LimitCount: int32(limit)}) + + if err != nil { + return nil, convertDBError(err) + } + + resultChats := make([]*model.Chat, len(chats)) + for index, sqlcChat := range chats { + memberIDs := make([]string, len(sqlcChat.MemberIds)) + + for i, memberID := range sqlcChat.MemberIds { + memberIDs[i] = memberID.String() + } + + resultChats[index] = &model.Chat{ + ID: sqlcChat.ID.String(), + MemberIDs: memberIDs, + CreatedAt: sqlcChat.CreatedAt.Time, + Title: sqlcChat.Title, + } + } + + return resultChats, nil +} + +func mapChatSqlcToModel(sqlcChat *sqlc.GetChatByIDRow) *model.Chat { + + memberIDs := make([]string, len(sqlcChat.MemberIds)) + + for i, memberID := range sqlcChat.MemberIds { + memberIDs[i] = memberID.String() + } + + return &model.Chat{ + ID: sqlcChat.ID.String(), + MemberIDs: memberIDs, + CreatedAt: sqlcChat.CreatedAt.Time, + Title: sqlcChat.Title, + } + +} diff --git a/internal/repository/postgres/errors.go b/internal/repository/postgres/errors.go new file mode 100644 index 00000000..b9897742 --- /dev/null +++ b/internal/repository/postgres/errors.go @@ -0,0 +1,46 @@ +package postgres + +import ( + "context" + "errors" + "fmt" + "highload/internal/repository" + "io" + "net" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +func convertDBError(err error) error { + var pgErr *pgconn.PgError + var netErr *net.OpError + var dnsErr *net.DNSError + + switch { + case errors.Is(err, context.DeadlineExceeded): + return fmt.Errorf("%w: %v", repository.ErrServiceUnavailable, err) + case errors.Is(err, pgx.ErrNoRows): + return repository.ErrNotFound + case errors.As(err, &pgErr): + switch pgErr.Code { + case "23505": + return repository.ErrAlreadyExists + case "53300", "57P03", "08006", "08001", "57P01", "57P02": + return repository.ErrServiceUnavailable + } + case errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, pgconn.ErrConnClosed): + return repository.ErrServiceUnavailable + case errors.Is(err, repository.ErrInvalidInput): + return repository.ErrInvalidInput + case errors.As(err, &netErr): + return repository.ErrServiceUnavailable + case errors.As(err, &dnsErr): + return repository.ErrServiceUnavailable + } + + return fmt.Errorf("%w: %v", repository.ErrDatabase, err) + +} diff --git a/internal/repository/postgres/message.go b/internal/repository/postgres/message.go new file mode 100644 index 00000000..ac205d3b --- /dev/null +++ b/internal/repository/postgres/message.go @@ -0,0 +1,205 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "highload/internal/repository" + "slices" + "time" + + "github.com/jackc/pgx/v5/pgtype" + "github.com/jackc/pgx/v5/pgxpool" +) + +type MessageRepository struct { + queries *sqlc.Queries + pool *pgxpool.Pool +} + +func NewMessageRepository(queries *sqlc.Queries, pool *pgxpool.Pool) *MessageRepository { + return &MessageRepository{ + queries: queries, + pool: pool, + } +} + +func (r *MessageRepository) CreateMessage(ctx context.Context, chatID, senderID, text, key string, attachmentIDs []string) (*model.Message, error) { + + uuidChatID, err := getUUIDFromString(chatID) + if err != nil { + return nil, convertDBError(err) + } + + txCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + uuidSenderID, err := getUUIDFromString(senderID) + if err != nil { + return nil, convertDBError(err) + } + + tx, err := r.pool.Begin(txCtx) + if err != nil { + return nil, convertDBError(err) + } + + defer func() { + rollbackCtx, rollbackCancel := context.WithTimeout( + context.WithoutCancel(ctx), + time.Second, + ) + defer rollbackCancel() + + _ = tx.Rollback(rollbackCtx) + }() + + txQueries := r.queries.WithTx(tx) + + sqlcMessage, err := txQueries.CreateMessage(txCtx, sqlc.CreateMessageParams{ChatID: *uuidChatID, SenderID: *uuidSenderID, Text: text}) + + if err != nil { + return nil, convertDBError(err) + } + + requestedIDs := make([]string, 0, len(attachmentIDs)) + for index, attID := range attachmentIDs { + attUUID, err := getUUIDFromString(attID) + if err != nil { + return nil, convertDBError(err) + } + + requestedIDs = append(requestedIDs, attUUID.String()) + + cmaParams := sqlc.CreateMessagesAttachmentsParams{MessageID: sqlcMessage.ID, AttachmentID: *attUUID, ChatID: sqlcMessage.ChatID, Position: int16(index)} + rowsAffected, err := txQueries.CreateMessagesAttachments(txCtx, cmaParams) + if err != nil { + return nil, convertDBError(err) + } + if rowsAffected == 0 { + return nil, repository.ErrNotFound + } + + } + + rowsAffected, err := txQueries.InsertMessageIdempotency(txCtx, sqlc.InsertMessageIdempotencyParams{UserID: sqlcMessage.SenderID, ChatID: sqlcMessage.ChatID, IdempotencyKey: key, MessageID: sqlcMessage.ID}) + if err != nil { + return nil, convertDBError(err) + } + + resultIDs := requestedIDs + + switch rowsAffected { + case 1: + err := tx.Commit(txCtx) + if err != nil { + return nil, convertDBError(err) + } + case 0: + rollbackCtx, rollbackCancel := context.WithTimeout(context.WithoutCancel(ctx), time.Second) + err := tx.Rollback(rollbackCtx) + rollbackCancel() + if err != nil { + return nil, convertDBError(err) + } + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + sqlcMessage, err = r.queries.GetIdempotentMessage(dbCtx, sqlc.GetIdempotentMessageParams{UserID: sqlcMessage.SenderID, ChatID: sqlcMessage.ChatID, IdempotencyKey: key}) + if err != nil { + return nil, convertDBError(err) + } + + rows, err := r.queries.GetMessagesAttachments(dbCtx, sqlcMessage.ID) + if err != nil { + return nil, convertDBError(err) + } + savedIDs := make([]string, 0, len(rows)) + + for _, row := range rows { + savedIDs = append(savedIDs, row.AttachmentID.String()) + } + + if sqlcMessage.Text != text || !slices.Equal(savedIDs, requestedIDs) { + return nil, repository.ErrConflict + } + resultIDs = savedIDs + + default: + return nil, repository.ErrServerError + } + + message := &model.Message{ + ID: sqlcMessage.ID.String(), + ChatID: sqlcMessage.ChatID.String(), + SenderID: sqlcMessage.SenderID.String(), + Text: sqlcMessage.Text, + AttachmentIDs: resultIDs, + CreatedAt: sqlcMessage.CreatedAt.Time, + } + + return message, nil + +} + +func (r *MessageRepository) GetMessagesByChatID(ctx context.Context, chatID string, limit int, after *repository.MessagePosition) ([]*model.Message, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidChatID, err := getUUIDFromString(chatID) + + if err != nil { + return nil, convertDBError(err) + } + + afterCreatedAt := pgtype.Timestamptz{Valid: false} + afterID := pgtype.UUID{Valid: false} + + if after != nil { + afterCreatedAt = pgtype.Timestamptz{Time: after.CreatedAt, Valid: true} + if uuidFromString, err := getUUIDFromString(after.ID); err == nil { + afterID = *uuidFromString + } else { + return nil, convertDBError(err) + } + } + + sqlcMessages, err := r.queries.GetMessagesByChatID(dbCtx, sqlc.GetMessagesByChatIDParams{ChatID: *uuidChatID, AfterID: afterID, AfterCreatedAt: afterCreatedAt, LimitCount: int32(limit)}) + + if err != nil { + return nil, convertDBError(err) + } + + messageIDs := make([]pgtype.UUID, len(sqlcMessages)) + for index, _ := range sqlcMessages { + messageIDs[index] = sqlcMessages[index].ID + } + + messageRow, err := r.queries.GetAttachmentsForMessages(dbCtx, messageIDs) + + if err != nil { + return nil, convertDBError(err) + } + + mapMessagesAtt := make(map[string][]string) + for index, _ := range messageRow { + messageID := messageRow[index].MessageID.String() + attachmentID := messageRow[index].AttachmentID.String() + + mapMessagesAtt[messageID] = append(mapMessagesAtt[messageID], attachmentID) + + } + + messages := make([]*model.Message, len(sqlcMessages)) + + for index, sqlcMessage := range sqlcMessages { + messages[index] = &model.Message{ + ID: sqlcMessage.ID.String(), + ChatID: sqlcMessage.ChatID.String(), + SenderID: sqlcMessage.SenderID.String(), + Text: sqlcMessage.Text, + AttachmentIDs: mapMessagesAtt[sqlcMessage.ID.String()], + CreatedAt: sqlcMessage.CreatedAt.Time} + } + + return messages, nil + +} diff --git a/internal/repository/postgres/user.go b/internal/repository/postgres/user.go new file mode 100644 index 00000000..d30dc071 --- /dev/null +++ b/internal/repository/postgres/user.go @@ -0,0 +1,85 @@ +package postgres + +import ( + "context" + "highload/internal/database/sqlc" + "highload/internal/model" + "time" +) + +type UserRepository struct { + queries *sqlc.Queries +} + +func NewUserRepository(queries *sqlc.Queries) *UserRepository { + return &UserRepository{queries: queries} +} + +func (r *UserRepository) Create(ctx context.Context, username, displayName string, passwordHash string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + sqlcUser, err := r.queries.CreateUser(dbCtx, sqlc.CreateUserParams{Username: username, DisplayName: displayName, PasswordHash: passwordHash}) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil + +} + +func (r *UserRepository) GetUserByID(ctx context.Context, id string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + uuidID, err := getUUIDFromString(id) + + if err != nil { + return nil, convertDBError(err) + } + sqlcUser, err := r.queries.GetUserByID(dbCtx, *uuidID) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil + +} + +func (r *UserRepository) GetUserByUsername(ctx context.Context, username string) (*model.User, error) { + dbCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + sqlcUser, err := r.queries.GetUserByUsername(dbCtx, username) + + if err != nil { + return nil, convertDBError(err) + } + + user := &model.User{ + ID: sqlcUser.ID.String(), + Username: sqlcUser.Username, + DisplayName: sqlcUser.DisplayName, + CreatedAt: sqlcUser.CreatedAt.Time, + PasswordHash: sqlcUser.PasswordHash, + } + + return user, nil +} diff --git a/internal/repository/postgres/utils.go b/internal/repository/postgres/utils.go new file mode 100644 index 00000000..e60cc7fb --- /dev/null +++ b/internal/repository/postgres/utils.go @@ -0,0 +1,17 @@ +package postgres + +import ( + "highload/internal/repository" + + "github.com/jackc/pgx/v5/pgtype" +) + +func getUUIDFromString(id string) (*pgtype.UUID, error) { + var uuidID pgtype.UUID + + if err := uuidID.Scan(id); err != nil { + return nil, repository.ErrInvalidInput + } + + return &uuidID, nil +} diff --git a/internal/repository/redis/errors.go b/internal/repository/redis/errors.go new file mode 100644 index 00000000..8453fc0e --- /dev/null +++ b/internal/repository/redis/errors.go @@ -0,0 +1,38 @@ +package redis + +import ( + "context" + "errors" + "fmt" + "highload/internal/repository" + "io" + "net" + + "github.com/redis/go-redis/v9" +) + +func convertDBError(err error) error { + + category := repository.ErrDatabase + + var netErr *net.OpError + + if errors.Is(err, redis.Nil) { + category = repository.ErrNotFound + } + if errors.As(err, &netErr) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, context.DeadlineExceeded) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, redis.ErrPoolTimeout) { + category = repository.ErrServiceUnavailable + } + if errors.Is(err, io.EOF) || + errors.Is(err, io.ErrUnexpectedEOF) { + category = repository.ErrServiceUnavailable + } + + return fmt.Errorf("%w: %v", category, err) +} diff --git a/internal/repository/redis/session.go b/internal/repository/redis/session.go new file mode 100644 index 00000000..bab4db7e --- /dev/null +++ b/internal/repository/redis/session.go @@ -0,0 +1,127 @@ +package redis + +import ( + "context" + "encoding/hex" + "encoding/json/v2" + "fmt" + "highload/internal/model" + "highload/internal/repository" + "time" + + "github.com/redis/go-redis/v9" +) + +type SessionRepository struct { + client *redis.Client +} + +func NewSessionRepository(client *redis.Client) *SessionRepository { + return &SessionRepository{client: client} +} + +type sessionData struct { + UserID string `json:"user_id"` + CreatedAt time.Time `json:"created_at"` + ExpiresAt time.Time `json:"expires_at"` +} + +func (r *SessionRepository) Create(ctx context.Context, tokenHash []byte, userID string, sessionTTL time.Duration) (*model.Session, error) { + key := "session:" + hex.EncodeToString(tokenHash) + now := time.Now().UTC() + data := sessionData{ + UserID: userID, + CreatedAt: now, + ExpiresAt: now.Add(sessionTTL), + } + value, err := json.Marshal(data) + + if err != nil { + return nil, fmt.Errorf("%w: encoding session %v", repository.ErrDatabase, err) + + } + + err = r.client.Set(ctx, key, value, sessionTTL).Err() + + if err != nil { + return nil, convertDBError(err) + } + + session := &model.Session{ + TokenHash: tokenHash, + UserID: data.UserID, + CreatedAt: data.CreatedAt, + ExpiresAt: data.ExpiresAt, + RevokedAt: nil, + } + + return session, nil + +} + +func (r *SessionRepository) GetSessionByTokenHash(ctx context.Context, tokenHash []byte) (*model.Session, error) { + key := "session:" + hex.EncodeToString(tokenHash) + + value, err := r.client.Get(ctx, key).Result() + if err != nil { + return nil, convertDBError(err) + } + + var data sessionData + + err = json.Unmarshal([]byte(value), &data) + + if err != nil { + return nil, fmt.Errorf("%w: decode session %v", repository.ErrDatabase, err) + + } + + session := &model.Session{ + TokenHash: tokenHash, + UserID: data.UserID, + CreatedAt: data.CreatedAt, + ExpiresAt: data.ExpiresAt, + RevokedAt: nil, + } + + return session, nil +} +func (r *SessionRepository) RevokeSession( + ctx context.Context, + tokenHash []byte, +) (int, error) { + key := "session:" + hex.EncodeToString(tokenHash) + + conn := r.client.Conn() + defer conn.Close() + + deleted, err := conn.Del(ctx, key).Result() + if err != nil { + return 0, convertDBError(err) + + } + + result, err := conn.Do(ctx, "WAITAOF", 1, 0, 2000).Slice() + if err != nil { + return 0, convertDBError(err) + + } + + if len(result) != 2 { + return 0, fmt.Errorf( + "%w: WAITAOF: expected 2 counters, got %d", + repository.ErrDatabase, len(result), + ) + } + + local, ok := result[0].(int64) + + if !ok { + return 0, fmt.Errorf("%w: expected int64, got %T", repository.ErrDatabase, result[0]) + } + if local != 1 { + return 0, fmt.Errorf("%w: WAITAOF expected local fsync = 1, got %d", repository.ErrServiceUnavailable, local) + } + + return int(deleted), nil +} diff --git a/internal/repository/session.go b/internal/repository/session.go new file mode 100644 index 00000000..01775c08 --- /dev/null +++ b/internal/repository/session.go @@ -0,0 +1,13 @@ +package repository + +import ( + "context" + "highload/internal/model" + "time" +) + +type SessionRepository interface { + Create(ctx context.Context, tokenHash []byte, userID string, sessionTTL time.Duration) (*model.Session, error) + GetSessionByTokenHash(ctx context.Context, tokenHash []byte) (*model.Session, error) + RevokeSession(ctx context.Context, tokenHash []byte) (int, error) +} diff --git a/internal/repository/user.go b/internal/repository/user.go new file mode 100644 index 00000000..a55d99f7 --- /dev/null +++ b/internal/repository/user.go @@ -0,0 +1,12 @@ +package repository + +import ( + "context" + "highload/internal/model" +) + +type UserRepository interface { + Create(ctx context.Context, username, displayName string, passwordHash string) (*model.User, error) + GetUserByID(ctx context.Context, id string) (*model.User, error) + GetUserByUsername(ctx context.Context, username string) (*model.User, error) +} diff --git a/internal/security/password.go b/internal/security/password.go new file mode 100644 index 00000000..d9fac229 --- /dev/null +++ b/internal/security/password.go @@ -0,0 +1,143 @@ +package security + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strconv" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + memory = 64 * 1024 + iterations = 3 + parallelism = 2 + saltLength = 16 + hashLength = 32 +) + +var ErrInvalidHash = errors.New("invalid password hash") + +func HashPassword(password string) (string, error) { + + salt := make([]byte, saltLength) + + if _, err := rand.Read(salt); err != nil { + return "", err + } + encodedSalt := base64.RawStdEncoding.EncodeToString(salt) + + hash := argon2.IDKey( + []byte(password), + salt, + iterations, + memory, + parallelism, + hashLength, + ) + + encodedHash := base64.RawStdEncoding.EncodeToString(hash) + + hashPassword := fmt.Sprintf( + "$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + memory, + iterations, + parallelism, + encodedSalt, + encodedHash, + ) + + return hashPassword, nil + +} + +func VerifyPassword(password, encodedHash string) (bool, error) { + parts := strings.Split(encodedHash, "$") + + if len(parts) != 6 || parts[0] != "" || parts[1] != "argon2id" { + return false, ErrInvalidHash + } + versionStr, ok := strings.CutPrefix(parts[2], "v=") + if !ok { + return false, ErrInvalidHash + } + + version, err := strconv.ParseUint(versionStr, 10, 32) + if err != nil || version != uint64(argon2.Version) { + return false, ErrInvalidHash + } + + params := strings.Split(parts[3], ",") + if len(params) != 3 { + return false, ErrInvalidHash + } + + parseParam := func(value, prefix string, bits int) (uint64, error) { + number, ok := strings.CutPrefix(value, prefix) + if !ok { + return 0, ErrInvalidHash + } + + return strconv.ParseUint(number, 10, bits) + } + + m, err := parseParam(params[0], "m=", 32) + if err != nil { + return false, ErrInvalidHash + } + + t, err := parseParam(params[1], "t=", 32) + if err != nil { + return false, ErrInvalidHash + } + + p, err := parseParam(params[2], "p=", 8) + if err != nil { + return false, ErrInvalidHash + } + + memory := uint32(m) + iterations := uint32(t) + parallelism := uint8(p) + + if memory < 8*uint32(parallelism) || + memory > 256*1024 || + iterations < 1 || iterations > 10 || + parallelism < 1 || parallelism > 8 { + return false, ErrInvalidHash + } + + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, ErrInvalidHash + } + + if len(salt) != saltLength { + return false, ErrInvalidHash + } + expectedHash, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, ErrInvalidHash + } + + if len(expectedHash) != hashLength { + return false, ErrInvalidHash + } + + computedHash := argon2.IDKey( + []byte(password), + salt, + iterations, + memory, + parallelism, + uint32(len(expectedHash)), + ) + + return subtle.ConstantTimeCompare(computedHash, expectedHash) == 1, nil + +} diff --git a/internal/service/attachment.go b/internal/service/attachment.go new file mode 100644 index 00000000..d000daff --- /dev/null +++ b/internal/service/attachment.go @@ -0,0 +1,204 @@ +package service + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/storage" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +const MaxAttachmentBytes = 10 * 1024 * 1024 +const maxImagePixels = 40_000_000 +const attachmentObjectPrefix = "attachments/" + +type AttachmentService struct { + repo repository.AttachmentRepository + chatService *ChatService + objects storage.ObjectWriter +} + +func NewAttachmentService(repo repository.AttachmentRepository, chatService *ChatService, objects storage.ObjectWriter) *AttachmentService { + return &AttachmentService{repo: repo, chatService: chatService, objects: objects} +} + +func (s *AttachmentService) CreateAttachment(ctx context.Context, chatID, userID, filename, declaredContentType string, body io.ReadSeeker, size int64) (*model.Attachment, error) { + + chat, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, err + } + + mimeType, err := validateAttachment(filename, declaredContentType, body, size) + + if err != nil { + return nil, err + } + + key := attachmentObjectPrefix + uuid.NewString() + + if err := ctx.Err(); err != nil { + return nil, err + } + sum := sha256.New() + n, err := io.Copy(sum, io.LimitReader(body, size+1)) + if err != nil || n != size { + return nil, ErrServerError + } + if _, err := body.Seek(0, io.SeekStart); err != nil { + return nil, ErrServerError + } + sumString := hex.EncodeToString(sum.Sum(nil)) + + var attachment *model.Attachment + fn := func(ctx context.Context, tx repository.AttachmentTx) error { + err := s.objects.Put(ctx, key, mimeType, body, size) + + if err != nil { + return convertStorageError(err) + } + + params := repository.CreateAttachmentParams{ + ChatID: chat.ID, + OwnerID: userID, + Filename: filename, + ObjectKey: key, + ContentType: mimeType, + Sha256: sumString, + SizeBytes: size, + } + attachment, err = tx.CreateAttachment(ctx, params) + + if err != nil { + return err + } + return nil + } + + acquired, err := s.repo.WithObjectKeyLock(ctx, key, fn) + + if err != nil { + return nil, convertRepoError(err) + } + + if !acquired { + return nil, ErrServiceUnavailable + } + + return attachment, nil + +} + +func (s *AttachmentService) GetAttachment(ctx context.Context, userID, attachmentID string) (*model.Attachment, error) { + _, err := uuid.Parse(attachmentID) + if err != nil { + return nil, ErrInvalidInput + } + + _, err = uuid.Parse(userID) + if err != nil { + return nil, ErrInvalidInput + } + + attachment, err := s.repo.GetAttachmentForUser(ctx, userID, attachmentID) + if err != nil { + return nil, convertRepoError(err) + } + + return attachment, nil +} + +func (s *AttachmentService) GetAttachmentContent(ctx context.Context, userID, attachmentID string) (*model.Attachment, io.ReadCloser, error) { + + attachment, err := s.GetAttachment(ctx, userID, attachmentID) + + if err != nil { + return nil, nil, err + } + + contentReader, err := s.objects.Get(ctx, attachment.ObjectKey) + if err != nil { + return nil, nil, convertStorageError(err) + } + + return attachment, contentReader, nil + +} + +func validateAttachment(filename, declaredContentType string, body io.ReadSeeker, size int64) (string, error) { + if !utf8.ValidString(filename) || strings.TrimSpace(filename) == "" || utf8.RuneCountInString(filename) > 255 { + return "", ErrInvalidInput + } + + if strings.ContainsAny(filename, "/\\") || containsControlChars(filename) { + return "", ErrInvalidInput + } + + if size > MaxAttachmentBytes { + return "", ErrPayloadTooLarge + } + if size <= 0 { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + imageConfig, format, err := image.DecodeConfig(io.LimitReader(body, size)) + if err != nil { + return "", ErrUnsupportMediaType + } + + if imageConfig.Height <= 0 || imageConfig.Width <= 0 { + return "", ErrUnsupportMediaType + } + if imageConfig.Width > maxImagePixels/imageConfig.Height { + return "", ErrPayloadTooLarge + } + + var mimeType string + switch format { + case "png": + mimeType = "image/png" + case "jpeg": + mimeType = "image/jpeg" + default: + return "", ErrUnsupportMediaType + + } + if mimeType != declaredContentType { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + _, _, err = image.Decode(io.LimitReader(body, size)) + if err != nil { + return "", ErrUnsupportMediaType + } + + if _, err := body.Seek(0, io.SeekStart); err != nil { + return "", ErrServerError + } + return mimeType, nil +} + +func containsControlChars(s string) bool { + for _, r := range s { + if (r >= 0 && r <= 31) || r == 127 { + return true + } + } + return false +} diff --git a/internal/service/attachment_cleanup.go b/internal/service/attachment_cleanup.go new file mode 100644 index 00000000..dd7cfc01 --- /dev/null +++ b/internal/service/attachment_cleanup.go @@ -0,0 +1,98 @@ +package service + +import ( + "context" + "highload/internal/repository" + "highload/internal/storage" + "strings" + "time" +) + +type AttachmentCleanupOptions struct { + GracePeriod time.Duration + DryRun bool + ListTimeout time.Duration +} + +type AttachmentCleanupService struct { + repo repository.AttachmentRepository + objects storage.ObjectCleanup + options *AttachmentCleanupOptions +} + +type OrphanReport struct { + Pages int `json:"s3_pages"` + Objects int `json:"objects_scanned"` + Checked int `json:"checked_in_db"` + Busy int `json:"busy"` + Candidates int `json:"orphan_candidates"` + Deleted int `json:"objects_deleted"` +} + +func NewAttachmentCleanupService(repo repository.AttachmentRepository, objects storage.ObjectCleanup, options *AttachmentCleanupOptions) *AttachmentCleanupService { + return &AttachmentCleanupService{repo: repo, objects: objects, options: options} +} + +func (s *AttachmentCleanupService) RunOnce(ctx context.Context) error { + _, err := s.ScanOrphans(ctx) + return err +} + +func (s *AttachmentCleanupService) ScanOrphans(ctx context.Context) (OrphanReport, error) { + var report OrphanReport + cutoff := time.Now().Add(-s.options.GracePeriod) + var token *string + + for { + if err := ctx.Err(); err != nil { + return report, err + } + + listCtx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + objects, nextToken, err := s.objects.ListPage(listCtx, token, attachmentObjectPrefix) + cancel() + if err != nil { + return report, err + } + report.Pages++ + + for _, obj := range objects { + report.Objects++ + if !strings.HasPrefix(obj.Key, attachmentObjectPrefix) || + obj.LastModified.IsZero() || !obj.LastModified.Before(cutoff) { + continue + } + + acquired, err := s.repo.WithObjectKeyLock(ctx, obj.Key, func(txCtx context.Context, txRepo repository.AttachmentTx) error { + exists, err := txRepo.ExistsByObjectKey(txCtx, obj.Key) + if err != nil { + return err + } + report.Checked++ + if exists { + return nil + } + report.Candidates++ + if s.options.DryRun { + return nil + } + if err := s.objects.Delete(txCtx, obj.Key); err != nil { + return err + } + report.Deleted++ + return nil + }) + if err != nil { + return report, err + } + if !acquired { + report.Busy++ + } + } + + if nextToken == nil { + return report, ctx.Err() + } + token = nextToken + } +} diff --git a/internal/service/attachment_reconcile.go b/internal/service/attachment_reconcile.go new file mode 100644 index 00000000..9cd554c1 --- /dev/null +++ b/internal/service/attachment_reconcile.go @@ -0,0 +1,162 @@ +package service + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "highload/internal/repository" + "highload/internal/storage" + "io" + "strings" + "time" +) + +const attachmentReconcilePageSize = 200 +const missingSampleLimit = 32 + +type attachmentReconcileStorage interface { + Check(context.Context) error + Head(context.Context, string) error + PutIfAbsent(context.Context, string, string, io.ReadSeeker, int64) error +} + +type AttachmentReconciler struct { + repo repository.AttachmentReconcileRepository + objects attachmentReconcileStorage + options AttachmentCleanupOptions +} + +type MissingReport struct { + Pages int `json:"db_pages"` + Rows int `json:"rows_scanned"` + Heads int `json:"s3_head_attempts"` + Missing int `json:"missing_objects"` + SampleIDs []string `json:"missing_attachment_ids"` +} + +func NewAttachmentReconciler(repo repository.AttachmentReconcileRepository, objects attachmentReconcileStorage, options AttachmentCleanupOptions) *AttachmentReconciler { + return &AttachmentReconciler{repo: repo, objects: objects, options: options} +} + +// ScanMissing checks old committed attachment rows using S3 metadata requests. +func (s *AttachmentReconciler) ScanMissing(ctx context.Context) (MissingReport, error) { + report := MissingReport{SampleIDs: []string{}} + cutoff := time.Now().Add(-s.options.GracePeriod) + if err := s.checkBucket(ctx); err != nil { + return report, err + } + var afterKey string + for { + rows, err := s.repo.ListAttachmentKeys(ctx, afterKey, attachmentReconcilePageSize) + if err != nil { + return report, err + } + report.Pages++ + for _, row := range rows { + report.Rows++ + if !strings.HasPrefix(row.ObjectKey, attachmentObjectPrefix) || + row.CreatedAt.IsZero() || !row.CreatedAt.Before(cutoff) { + continue + } + report.Heads++ + present, err := s.objectPresent(ctx, row.ObjectKey) + if err != nil { + return report, err + } + if !present { + report.Missing++ + if len(report.SampleIDs) < missingSampleLimit { + report.SampleIDs = append(report.SampleIDs, row.ID) + } + } + } + if len(rows) < attachmentReconcilePageSize { + return report, ctx.Err() + } + afterKey = rows[len(rows)-1].ObjectKey + } +} + +// Restore verifies the original bytes before writing a missing object under its existing key. +func (s *AttachmentReconciler) Restore(ctx context.Context, id string, original io.Reader) (string, error) { + attachment, err := s.repo.GetAttachmentByID(ctx, id) + if err != nil { + return "", err + } + if !strings.HasPrefix(attachment.ObjectKey, attachmentObjectPrefix) || + attachment.CreatedAt.IsZero() || !attachment.CreatedAt.Before(time.Now().Add(-s.options.GracePeriod)) { + return "", errors.New("attachment is outside the restore scope or still within grace") + } + if attachment.SizeBytes <= 0 || attachment.SizeBytes > MaxAttachmentBytes { + return "", errors.New("invalid original size in attachment metadata") + } + if err := s.checkBucket(ctx); err != nil { + return "", err + } + + // Read at most one original. Keep the verified bytes stable until Put completes. + data, err := io.ReadAll(io.LimitReader(original, attachment.SizeBytes+1)) + if err != nil { + return "", fmt.Errorf("read backup original: %w", err) + } + sum := sha256.Sum256(data) + if int64(len(data)) != attachment.SizeBytes || hex.EncodeToString(sum[:]) != attachment.SHA256 { + return "", errors.New("backup original does not match the stored size and SHA-256") + } + + var result string + acquired, err := s.repo.WithObjectKeyLock(ctx, attachment.ObjectKey, func(txCtx context.Context, txRepo repository.AttachmentTx) error { + current, err := txRepo.GetAttachmentByID(txCtx, id) + if err != nil { + return err + } + if current.ObjectKey != attachment.ObjectKey || current.SizeBytes != attachment.SizeBytes || + current.SHA256 != attachment.SHA256 || current.ContentType != attachment.ContentType || + !current.CreatedAt.Equal(attachment.CreatedAt) { + return errors.New("attachment metadata changed during restore") + } + present, err := s.objectPresent(txCtx, current.ObjectKey) + if err != nil { + return err + } + if present { + result = "already_present" + return nil + } + if s.options.DryRun { + result = "would_restore" + return nil + } + if err := s.objects.PutIfAbsent(txCtx, current.ObjectKey, current.ContentType, bytes.NewReader(data), current.SizeBytes); err != nil { + return err + } + result = "restored" + return nil + }) + if err != nil { + return "", err + } + if !acquired { + return "", errors.New("attachment key is busy; retry the restore") + } + return result, nil +} + +func (s *AttachmentReconciler) checkBucket(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + defer cancel() + return s.objects.Check(ctx) +} + +func (s *AttachmentReconciler) objectPresent(ctx context.Context, key string) (bool, error) { + headCtx, cancel := context.WithTimeout(ctx, s.options.ListTimeout) + defer cancel() + err := s.objects.Head(headCtx, key) + if errors.Is(err, storage.ErrObjectNotFound) { + return false, nil + } + return err == nil, err +} diff --git a/internal/service/auth.go b/internal/service/auth.go new file mode 100644 index 00000000..a9cfeeca --- /dev/null +++ b/internal/service/auth.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/security" + "time" +) + +const tokenType = "Bearer" + +type AuthService struct { + userRepo repository.UserRepository + sessionRepo repository.SessionRepository + sessionTTL time.Duration +} + +func NewAuthService(userRepo repository.UserRepository, sessionRepo repository.SessionRepository, sessionTTL time.Duration) *AuthService { + return &AuthService{ + userRepo: userRepo, + sessionRepo: sessionRepo, + sessionTTL: sessionTTL, + } +} + +func (s *AuthService) Authenticate(ctx context.Context, username, password string) (*model.User, error) { + + userOK := checkUsername(username) + passwordOK := checkPassword(password) + + if !userOK || !passwordOK { + return nil, ErrUnauthorized + } + + user, err := s.userRepo.GetUserByUsername(ctx, username) + + if err != nil { + if errors.Is(err, repository.ErrNotFound) { + return nil, ErrUnauthorized + } + return nil, convertRepoError(err) + } + + isVerify, err := security.VerifyPassword(password, user.PasswordHash) + + if err != nil { + return nil, ErrServerError + } + + if !isVerify { + return nil, ErrUnauthorized + } + + return user, nil +} + +type CreatedSession struct { + Token string + TokenType string + ExpiresAt time.Time + User *model.User +} + +func (s *AuthService) CreateSession(ctx context.Context, username, password string) (*CreatedSession, error) { + user, err := s.Authenticate(ctx, username, password) + if err != nil { + return nil, err + } + + if s.sessionTTL <= 0 { + return nil, ErrServerError + } + + tokenBytes := make([]byte, 32) + _, err = rand.Read(tokenBytes) + if err != nil { + return nil, ErrServerError + } + + token := base64.RawURLEncoding.EncodeToString(tokenBytes) + + tokenHash := sha256.Sum256([]byte(token)) + + session, err := s.sessionRepo.Create(ctx, tokenHash[:], user.ID, s.sessionTTL) + + if err != nil { + return nil, convertRepoError(err) + } + + createdSession := &CreatedSession{ + Token: token, + TokenType: tokenType, + ExpiresAt: session.ExpiresAt, + User: user, + } + + return createdSession, nil + +} + +func (s *AuthService) ValidateSession(ctx context.Context, token string) (string, error) { + + if token == "" { + return "", ErrUnauthorized + } + + tokenHash := sha256.Sum256([]byte(token)) + + session, err := s.sessionRepo.GetSessionByTokenHash(ctx, tokenHash[:]) + if err != nil { + if errors.Is(err, repository.ErrNotFound) { + return "", ErrUnauthorized + } + return "", convertRepoError(err) + } + + if session.RevokedAt != nil { + return "", ErrUnauthorized + } + + if !time.Now().Before(session.ExpiresAt) { + return "", ErrUnauthorized + } + + return session.UserID, nil +} + +func (s *AuthService) RevokeSession(ctx context.Context, token string) error { + tokenHash := sha256.Sum256([]byte(token)) + + _, err := s.sessionRepo.RevokeSession(ctx, tokenHash[:]) + + if err != nil { + return convertRepoError(err) + } + return nil +} diff --git a/internal/service/chat.go b/internal/service/chat.go new file mode 100644 index 00000000..937eacd1 --- /dev/null +++ b/internal/service/chat.go @@ -0,0 +1,194 @@ +package service + +import ( + "context" + "errors" + "highload/internal/model" + "highload/internal/repository" + "slices" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +type ChatService struct { + repo repository.ChatRepository + userRepo repository.UserRepository + userService *UserService +} + +func NewChatService(repo repository.ChatRepository, userRepo repository.UserRepository, userService *UserService) *ChatService { + return &ChatService{repo: repo, userRepo: userRepo, userService: userService} +} + +func (s *ChatService) Create(ctx context.Context, title string, memberIDs []string, userID string) (*model.Chat, error) { + + trimTitle := strings.TrimSpace(title) + + lenTitle := utf8.RuneCountInString(title) + if lenTitle < 1 || lenTitle > 100 { + return nil, ErrInvalidInput + } + + if trimTitle == "" { + return nil, ErrInvalidInput + } + + if memberIDs == nil || len(memberIDs) == 0 || len(memberIDs) > 100 { + return nil, ErrInvalidInput + } + + seen := make(map[string]struct{}) + for _, memberID := range memberIDs { + parsedID, err := uuid.Parse(memberID) + + if err != nil { + return nil, ErrInvalidInput + } + + normalizedID := parsedID.String() + + if _, exist := seen[normalizedID]; exist { + return nil, ErrInvalidInput + } + + seen[normalizedID] = struct{}{} + + _, err = s.userRepo.GetUserByID(ctx, memberID) + if err != nil { + // NOTE: MemberNotFound –> InvalidInput + if errors.Is(err, repository.ErrNotFound) { + return nil, ErrInvalidInput + } + return nil, convertRepoError(err) + } + } + + parsedUserID, err := uuid.Parse(userID) + if err != nil { + return nil, ErrServerError + } + normalizedUserID := parsedUserID.String() + + _, creatorExists := seen[normalizedUserID] + if !creatorExists { + return nil, ErrInvalidInput + } + + chat, err := s.repo.CreateChat(ctx, title, memberIDs) + + if err != nil { + return nil, convertRepoError(err) + } + + return chat, nil +} + +func (s *ChatService) GetChatsByUserID(ctx context.Context, userID string, limit *int, cursor *string) ([]*model.Chat, *string, error) { + + actualLimit := 50 + + if limit != nil { + actualLimit = *limit + } + + if actualLimit <= 0 || actualLimit > 100 { + return nil, nil, ErrInvalidInput + } + + var chatPosition *repository.ChatPosition + + if cursor != nil { + chCursor, err := decodeCursor[chatCursor](*cursor) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + _, err = uuid.Parse(chCursor.ChatID) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + if chCursor.CreatedAt.IsZero() { + return nil, nil, ErrInvalidInput + } + + _, err = uuid.Parse(chCursor.UserID) + if err != nil { + return nil, nil, ErrInvalidInput + } + + if chCursor.UserID != userID { + return nil, nil, ErrInvalidInput + } + + chatPosition = &repository.ChatPosition{ + CreatedAt: chCursor.CreatedAt, + ID: chCursor.ChatID, + } + } + + chats, err := s.repo.GetChatsByUserID(ctx, userID, actualLimit+1, chatPosition) + + if err != nil { + return nil, nil, convertRepoError(err) + } + + var chCursor *chatCursor + + if len(chats) > actualLimit { + chCursor = &chatCursor{ + ChatID: chats[actualLimit-1].ID, + UserID: userID, + CreatedAt: chats[actualLimit-1].CreatedAt, + } + cursor, err = encodeCursor[chatCursor](*chCursor) + if err != nil { + return nil, nil, ErrServerError + } + + return chats[:actualLimit], cursor, nil + } + + return chats, nil, nil + +} + +func (s *ChatService) GetChatByID(ctx context.Context, chatID, userID string) (*model.Chat, error) { + + _, err := uuid.Parse(chatID) + + if err != nil { + return nil, ErrInvalidInput + } + + _, err = uuid.Parse(userID) + + if err != nil { + return nil, ErrInvalidInput + } + + chat, err := s.repo.GetChatByID(ctx, chatID) + + if err != nil { + return nil, convertRepoError(err) + } + + index := slices.Index(chat.MemberIDs, userID) + + if index == -1 { + return nil, ErrNotFound + } + + return chat, nil +} + +type chatCursor struct { + CreatedAt time.Time `json:"created_at"` + ChatID string `json:"chat_id"` + UserID string `json:"user_id"` +} diff --git a/internal/service/errors.go b/internal/service/errors.go new file mode 100644 index 00000000..7c1be70b --- /dev/null +++ b/internal/service/errors.go @@ -0,0 +1,63 @@ +package service + +import ( + "errors" + "fmt" + "highload/internal/repository" + "highload/internal/storage" +) + +var ( + ErrInvalidInput = errors.New("invalid input") + ErrAlreadyExists = errors.New("user already exists") + ErrNotFound = errors.New("user not found") + ErrServerError = errors.New("server error") + ErrUnauthorized = errors.New("unauthorized") + ErrServiceUnavailable = errors.New("service unavailable") + ErrConflict = errors.New("data conflict") + ErrUnsupportMediaType = errors.New("unsupported mediaType") + ErrPayloadTooLarge = errors.New("payload too large") +) + +func convertRepoError(err error) error { + + var category error + + switch { + + case errors.Is(err, repository.ErrNotFound): + category = ErrNotFound + case errors.Is(err, repository.ErrAlreadyExists): + category = ErrAlreadyExists + case errors.Is(err, repository.ErrInvalidInput): + category = ErrInvalidInput + case errors.Is(err, repository.ErrDatabase) || errors.Is(err, repository.ErrServerError): + category = ErrServerError + case errors.Is(err, repository.ErrServiceUnavailable): + category = ErrServiceUnavailable + case errors.Is(err, repository.ErrConflict): + category = ErrConflict + default: + return err + } + + return fmt.Errorf("%w: %v", category, err) +} + +func convertStorageError(err error) error { + var category error + + switch { + case errors.Is(err, storage.ErrObjectNotFound): + category = ErrNotFound + case errors.Is(err, storage.ErrServiceUnavailable): + category = ErrServiceUnavailable + case errors.Is(err, storage.ErrStorage): + category = ErrServerError + default: + return err + } + + return fmt.Errorf("%w: %v", category, err) + +} diff --git a/internal/service/helper.go b/internal/service/helper.go new file mode 100644 index 00000000..764668cc --- /dev/null +++ b/internal/service/helper.go @@ -0,0 +1,63 @@ +package service + +import ( + "encoding/base64" + "encoding/json/v2" + "regexp" + "strings" + "unicode/utf8" +) + +func encodeCursor[C any](cursor C) (*string, error) { + data, err := json.Marshal(cursor) + + if err != nil { + return nil, err + } + encoded := base64.RawURLEncoding.EncodeToString(data) + + return &encoded, nil + +} + +func decodeCursor[C any](cursor string) (*C, error) { + data, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return nil, ErrInvalidInput + } + + var decoded C + + err = json.Unmarshal(data, &decoded) + + if err != nil { + return nil, ErrInvalidInput + } + + return &decoded, nil +} + +var usernameRegex = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +func checkUsername(username string) bool { + return usernameRegex.MatchString(username) +} + +func checkPassword(password string) bool { + + if !utf8.ValidString(password) { + return false + } + + if strings.TrimSpace(password) == "" { + return false + } + + passwordLength := utf8.RuneCountInString(password) + if passwordLength < 12 || passwordLength > 128 { + return false + } + + return true + +} diff --git a/internal/service/message.go b/internal/service/message.go new file mode 100644 index 00000000..dfcff6d6 --- /dev/null +++ b/internal/service/message.go @@ -0,0 +1,142 @@ +package service + +import ( + "context" + "highload/internal/model" + "highload/internal/repository" + "strings" + "time" + "unicode/utf8" + + "github.com/google/uuid" +) + +type MessageService struct { + repo repository.MessageRepository + userService *UserService + chatService *ChatService +} + +func NewMessageService(repo repository.MessageRepository, chatService *ChatService, userService *UserService) *MessageService { + return &MessageService{repo: repo, chatService: chatService, userService: userService} +} + +func (s *MessageService) CreateMessage(ctx context.Context, text, chatID, userID, key string, attachmentIDs []string) (*model.Message, error) { + + _, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, err + } + + lenText := utf8.RuneCountInString(text) + + if lenText > 4000 || (strings.TrimSpace(text) == "" && len(attachmentIDs) == 0) { + return nil, ErrInvalidInput + } + + if len(attachmentIDs) > 10 { + return nil, ErrInvalidInput + } + + seen := make(map[uuid.UUID]struct{}, len(attachmentIDs)) + for _, rawID := range attachmentIDs { + id, err := uuid.Parse(rawID) + if err != nil { + return nil, ErrInvalidInput + } + if _, exists := seen[id]; exists { + return nil, ErrInvalidInput + } + seen[id] = struct{}{} + + } + + message, err := s.repo.CreateMessage(ctx, chatID, userID, text, key, attachmentIDs) + + if err != nil { + return nil, convertRepoError(err) + } + + return message, nil +} + +func (s *MessageService) GetMessagesByChatID(ctx context.Context, chatID, userID string, limit *int, cursor *string) ([]*model.Message, *string, error) { + + actualLimit := 50 + + if limit != nil { + actualLimit = *limit + } + + if actualLimit <= 0 || actualLimit > 100 { + + return nil, nil, ErrInvalidInput + } + + chat, err := s.chatService.GetChatByID(ctx, chatID, userID) + + if err != nil { + return nil, nil, err + } + + var messagePosition *repository.MessagePosition + + if cursor != nil { + mesCursor, err := decodeCursor[messageCursor](*cursor) + + if err != nil { + return nil, nil, ErrInvalidInput + } + + if _, err := uuid.Parse(mesCursor.ChatID); err != nil { + return nil, nil, ErrInvalidInput + } + + if _, err := uuid.Parse(mesCursor.MessageID); err != nil { + return nil, nil, ErrInvalidInput + } + + if mesCursor.CreatedAt.IsZero() { + return nil, nil, ErrInvalidInput + } + + if mesCursor.ChatID != chat.ID { + return nil, nil, ErrInvalidInput + } + + messagePosition = &repository.MessagePosition{ + ID: mesCursor.MessageID, + CreatedAt: mesCursor.CreatedAt, + } + + } + + messages, err := s.repo.GetMessagesByChatID(ctx, chat.ID, actualLimit+1, messagePosition) + + if err != nil { + return nil, nil, convertRepoError(err) + } + + if len(messages) > actualLimit { + mesCursor := &messageCursor{ + MessageID: messages[actualLimit-1].ID, + ChatID: chat.ID, + CreatedAt: messages[actualLimit-1].CreatedAt, + } + cursor, err = encodeCursor[messageCursor](*mesCursor) + if err != nil { + return nil, nil, ErrServerError + } + return messages[:actualLimit], cursor, nil + } + + return messages, nil, nil + +} + +type messageCursor struct { + MessageID string `json:"message_id"` + ChatID string `json:"chat_id"` + CreatedAt time.Time `json:"created_at"` +} diff --git a/internal/service/user.go b/internal/service/user.go new file mode 100644 index 00000000..6556150a --- /dev/null +++ b/internal/service/user.go @@ -0,0 +1,70 @@ +package service + +import ( + "context" + "highload/internal/model" + "highload/internal/repository" + "highload/internal/security" + "strings" + "unicode/utf8" + + "github.com/google/uuid" +) + +type UserService struct { + repo repository.UserRepository +} + +func NewUserService(repo repository.UserRepository) *UserService { + return &UserService{repo} +} + +func (s *UserService) CreateUser(ctx context.Context, username, displayName, password string) (*model.User, error) { + + userOK := checkUsername(username) + passwordOK := checkPassword(password) + + if !userOK || !passwordOK { + return nil, ErrInvalidInput + } + + if !utf8.ValidString(displayName) { + return nil, ErrInvalidInput + } + if strings.TrimSpace(displayName) == "" { + return nil, ErrInvalidInput + } + + if utf8.RuneCountInString(displayName) > 100 { + return nil, ErrInvalidInput + } + + passwordHash, err := security.HashPassword(password) + if err != nil { + return nil, ErrServerError + } + + user, err := s.repo.Create(ctx, username, displayName, passwordHash) + if err != nil { + return nil, convertRepoError(err) + } + + return user, nil + +} + +func (s *UserService) GetUser(ctx context.Context, id string) (*model.User, error) { + + _, err := uuid.Parse(id) + if err != nil { + return nil, ErrInvalidInput + } + + user, err := s.repo.GetUserByID(ctx, id) + + if err != nil { + return nil, convertRepoError(err) + } + + return user, nil +} diff --git a/internal/storage/errors.go b/internal/storage/errors.go new file mode 100644 index 00000000..993ef825 --- /dev/null +++ b/internal/storage/errors.go @@ -0,0 +1,9 @@ +package storage + +import "errors" + +var ( + ErrServiceUnavailable = errors.New("object storage unavailable") + ErrStorage = errors.New("storage error") + ErrObjectNotFound = errors.New("object not found") +) diff --git a/internal/storage/object_cleanup_storage.go b/internal/storage/object_cleanup_storage.go new file mode 100644 index 00000000..43afe5b5 --- /dev/null +++ b/internal/storage/object_cleanup_storage.go @@ -0,0 +1,16 @@ +package storage + +import ( + "context" + "time" +) + +type ObjectCleanup interface { + ListPage(ctx context.Context, token *string, prefix string) ([]ObjectInfo, *string, error) + Delete(ctx context.Context, key string) error +} + +type ObjectInfo struct { + Key string + LastModified time.Time +} diff --git a/internal/storage/object_writer.go b/internal/storage/object_writer.go new file mode 100644 index 00000000..9c3b6239 --- /dev/null +++ b/internal/storage/object_writer.go @@ -0,0 +1,11 @@ +package storage + +import ( + "context" + "io" +) + +type ObjectWriter interface { + Put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error + Get(ctx context.Context, key string) (io.ReadCloser, error) +} diff --git a/internal/storage/s3/client.go b/internal/storage/s3/client.go new file mode 100644 index 00000000..04b7cc5a --- /dev/null +++ b/internal/storage/s3/client.go @@ -0,0 +1,160 @@ +package s3 + +import ( + "context" + "fmt" + "highload/internal/config" + "highload/internal/storage" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type Storage struct { + Client *awss3.Client + Bucket string +} + +func NewStorage(ctx context.Context, cfg *config.S3) (*Storage, error) { + provider := credentials.NewStaticCredentialsProvider( + cfg.AccessKeyID, + cfg.SecretAccessKey, + "", + ) + sdkConfig, err := awsconfig.LoadDefaultConfig( + ctx, + awsconfig.WithRegion(cfg.Region), + awsconfig.WithCredentialsProvider(provider), + ) + + if err != nil { + return nil, err + } + + client := awss3.NewFromConfig(sdkConfig, func(opts *awss3.Options) { + opts.BaseEndpoint = aws.String(cfg.Endpoint) + opts.UsePathStyle = true + }) + + return &Storage{ + Client: client, + Bucket: cfg.Bucket, + }, nil +} + +func (s *Storage) Check(ctx context.Context) error { + _, err := s.Client.HeadBucket(ctx, &awss3.HeadBucketInput{ + Bucket: aws.String(s.Bucket), + }) + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Head(ctx context.Context, key string) error { + _, err := s.Client.HeadObject(ctx, &awss3.HeadObjectInput{ + Bucket: &s.Bucket, + Key: &key, + }) + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error { + return s.put(ctx, key, contentType, body, size, nil) +} + +func (s *Storage) PutIfAbsent(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64) error { + return s.put(ctx, key, contentType, body, size, aws.String("*")) +} + +func (s *Storage) put(ctx context.Context, key, contentType string, body io.ReadSeeker, size int64, ifNoneMatch *string) error { + _, err := s.Client.PutObject(ctx, &awss3.PutObjectInput{ + Bucket: &s.Bucket, + Key: &key, + Body: body, + ContentLength: aws.Int64(size), + IfNoneMatch: ifNoneMatch, + ContentType: &contentType}) + + if err != nil { + return convertS3Error(err) + } + return nil +} + +func (s *Storage) Get(ctx context.Context, key string) (io.ReadCloser, error) { + params := &awss3.GetObjectInput{ + Bucket: &s.Bucket, + Key: &key, + } + objectOutput, err := s.Client.GetObject(ctx, params) + + if err != nil { + return nil, convertS3Error(err) + } + + return objectOutput.Body, nil + +} + +func (s *Storage) Delete(ctx context.Context, key string) error { + + params := &awss3.DeleteObjectInput{ + Bucket: &s.Bucket, + Key: &key, + } + _, err := s.Client.DeleteObject(ctx, params) + + if err != nil { + return convertS3Error(err) + } + + return nil + +} + +func (s *Storage) ListPage(ctx context.Context, token *string, prefix string) ([]storage.ObjectInfo, *string, error) { + input := awss3.ListObjectsV2Input{ + Bucket: &s.Bucket, + Prefix: aws.String(prefix), + ContinuationToken: token, + } + + output, err := s.Client.ListObjectsV2(ctx, &input) + + if err != nil { + return nil, nil, convertS3Error(err) + } + + var objects []storage.ObjectInfo + + for _, obj := range output.Contents { + if obj.Key == nil || obj.LastModified == nil || *obj.Key == "" { + continue + } + + objects = append(objects, storage.ObjectInfo{ + Key: *obj.Key, + LastModified: *obj.LastModified, + }) + } + if !aws.ToBool(output.IsTruncated) { + return objects, nil, nil + } + + nextToken := output.NextContinuationToken + + if nextToken == nil || aws.ToString(nextToken) == "" || aws.ToString(nextToken) == aws.ToString(token) { + return nil, nil, fmt.Errorf("%w: invalid next token", storage.ErrStorage) + } + + return objects, nextToken, nil +} diff --git a/internal/storage/s3/errors.go b/internal/storage/s3/errors.go new file mode 100644 index 00000000..353a969d --- /dev/null +++ b/internal/storage/s3/errors.go @@ -0,0 +1,35 @@ +package s3 + +import ( + "context" + "errors" + "fmt" + "highload/internal/storage" + + smithyhttp "github.com/aws/smithy-go/transport/http" +) + +func convertS3Error(err error) error { + category := storage.ErrStorage + + var sendErr *smithyhttp.RequestSendError + var responseErr *smithyhttp.ResponseError + + switch { + case errors.Is(err, context.Canceled): + //Ошибка storage.ErrStorage + case errors.Is(err, context.DeadlineExceeded) || errors.As(err, &sendErr): + category = storage.ErrServiceUnavailable + + case errors.As(err, &responseErr): + code := responseErr.HTTPStatusCode() + switch code { + case 404: + category = storage.ErrObjectNotFound + case 429, 500, 503, 504, 502: + category = storage.ErrServiceUnavailable + } + } + + return fmt.Errorf("%w: %v", category, err) +} diff --git a/internal/transport/http/handler/attachment.go b/internal/transport/http/handler/attachment.go new file mode 100644 index 00000000..9a42e241 --- /dev/null +++ b/internal/transport/http/handler/attachment.go @@ -0,0 +1,168 @@ +package handler + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "io" + "log/slog" + "net/http" + "os" + "time" +) + +type AttachmentHandler struct { + attachmentService *service.AttachmentService + uploadSlots chan struct{} +} + +func NewAttachmentHandler(attachmentService *service.AttachmentService) *AttachmentHandler { + uploadSlots := make(chan struct{}, 1) + return &AttachmentHandler{attachmentService: attachmentService, uploadSlots: uploadSlots} +} + +func (h *AttachmentHandler) CreateAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + chatID := r.PathValue("chat_id") + filename := r.URL.Query().Get("filename") + contentType := r.Header.Get("Content-Type") + + select { + case h.uploadSlots <- struct{}{}: + defer func() { <-h.uploadSlots }() + case <-time.After(1 * time.Second): + w.Header().Set("Retry-After", "1") + httputils.WriteError(w, http.StatusTooManyRequests, httputils.ErrorDetails{ + Code: "rate_limited", + Message: "too many concurrent uploads, try again later", + RequestID: requestID, + }) + return + + case <-r.Context().Done(): + return + } + + file, err := os.CreateTemp("", "attachment-*") + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + defer file.Close() + if err := os.Remove(file.Name()); err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + body := http.MaxBytesReader(w, r.Body, service.MaxAttachmentBytes) + defer body.Close() + size, err := io.Copy(file, uploadBodyReader{body}) + + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + httputils.WriteError(w, http.StatusRequestEntityTooLarge, httputils.ErrorDetails{ + Code: "payload_too_large", + Message: "payload_too_large", + RequestID: requestID, + }) + return + } + if err != nil { + var readErr *uploadBodyReadError + if !errors.As(err, &readErr) { + httputils.WriteServiceError(w, err, requestID) + return + } + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "invalid_request", + RequestID: requestID, + }) + return + } + + attachment, err := h.attachmentService.CreateAttachment(r.Context(), chatID, userID, filename, contentType, file, size) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertAttachmentModelToResponse(attachment) + wrErr := httputils.WriteJSON[attachmentResponse](w, http.StatusCreated, *response) + if wrErr != nil { + slog.Error("failed to write create attachment response", "error", wrErr, "requestID", requestID) + } + +} + +type uploadBodyReader struct{ io.Reader } + +func (r uploadBodyReader) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + if err != nil && err != io.EOF { + return n, &uploadBodyReadError{err} + } + return n, err +} + +type uploadBodyReadError struct{ cause error } + +func (e *uploadBodyReadError) Error() string { return e.cause.Error() } +func (e *uploadBodyReadError) Unwrap() error { return e.cause } + +func (h *AttachmentHandler) GetAttachment(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + attachmentID := r.PathValue("attachment_id") + + attachment, err := h.attachmentService.GetAttachment(r.Context(), userID, attachmentID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + wrErr := httputils.WriteJSON[attachmentResponse](w, http.StatusOK, *convertAttachmentModelToResponse(attachment)) + if wrErr != nil { + slog.Error("failed to write get attachment response", "error", wrErr, "requestID", requestID) + } +} + +func (h *AttachmentHandler) GetAttachmentContent(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + attachmentID := r.PathValue("attachment_id") + variant := r.URL.Query().Get("variant") + + if variant != "" && variant != "original" { + httputils.WriteError(w, http.StatusNotFound, httputils.ErrorDetails{ + Code: "not_found", + Message: "requested variant does not exist", + RequestID: requestID, + }) + return + } + + attachment, stream, err := h.attachmentService.GetAttachmentContent(r.Context(), userID, attachmentID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + defer stream.Close() + + w.Header().Set("Content-Type", attachment.ContentType) + + _, err = io.Copy(w, stream) + + if err != nil { + slog.Error("failed to copy attachment content to w", "error", err, "request_id", requestID) + } + +} diff --git a/internal/transport/http/handler/attachment_types.go b/internal/transport/http/handler/attachment_types.go new file mode 100644 index 00000000..e8132d54 --- /dev/null +++ b/internal/transport/http/handler/attachment_types.go @@ -0,0 +1,38 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type attachmentResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []string `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func convertAttachmentModelToResponse(attachment *model.Attachment) *attachmentResponse { + return &attachmentResponse{ + ID: attachment.ID, + ChatID: attachment.ChatID, + OwnerID: attachment.OwnerID, + Filename: attachment.Filename, + ContentType: attachment.ContentType, + SizeBytes: attachment.SizeBytes, + SHA256: attachment.SHA256, + Status: "ready", + Variants: []string{}, + ErrorCode: nil, + CreatedAt: httputils.FormatTime(attachment.CreatedAt), + UpdatedAt: httputils.FormatTime(attachment.UpdatedAt), + } +} diff --git a/internal/transport/http/handler/auth.go b/internal/transport/http/handler/auth.go new file mode 100644 index 00000000..78d46e5a --- /dev/null +++ b/internal/transport/http/handler/auth.go @@ -0,0 +1,94 @@ +package handler + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "io" + "net/http" +) + +type AuthHandler struct { + authService *service.AuthService +} + +func NewAuthHandler(authService *service.AuthService) *AuthHandler { + return &AuthHandler{authService: authService} +} + +func (h *AuthHandler) CreateSession(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + username, password, ok := r.BasicAuth() + + if !ok { + httputils.WriteBasicUnauthorized(w, requestID) + return + } + + var probe [1]byte + + n, err := r.Body.Read(probe[:]) + + if n > 0 || (err != nil && !errors.Is(err, io.EOF)) { + _ = httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "Request body must be empty", + RequestID: requestID, + }) + return + } + + createdSession, err := h.authService.CreateSession(r.Context(), username, password) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBasicUnauthorized(w, requestID) + return + } + + httputils.WriteServiceError(w, err, requestID) + return + } + + response := createSessionResponse{ + Token: createdSession.Token, + TokenType: createdSession.TokenType, + ExpiresAt: httputils.FormatTime(createdSession.ExpiresAt), + User: UserResponse{ + Username: createdSession.User.Username, + ID: createdSession.User.ID, + DisplayName: createdSession.User.DisplayName, + CreatedAt: httputils.FormatTime(createdSession.User.CreatedAt), + }, + } + + httputils.WriteJSON[createSessionResponse](w, http.StatusCreated, response) + +} + +func (h *AuthHandler) DeleteSession(w http.ResponseWriter, r *http.Request) { + + requestID := middleware.RequestIDFromContext(r.Context()) + + token, ok := httputils.ExtractBearerToken(r) + + if !ok { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + + err := h.authService.RevokeSession(r.Context(), token) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + httputils.WriteServiceError(w, err, requestID) + return + } + + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/transport/http/handler/auth_types.go b/internal/transport/http/handler/auth_types.go new file mode 100644 index 00000000..a1b0b2b8 --- /dev/null +++ b/internal/transport/http/handler/auth_types.go @@ -0,0 +1,15 @@ +package handler + +type UserResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type createSessionResponse struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt string `json:"expires_at"` + User UserResponse `json:"user"` +} diff --git a/internal/transport/http/handler/chat.go b/internal/transport/http/handler/chat.go new file mode 100644 index 00000000..9b98eac0 --- /dev/null +++ b/internal/transport/http/handler/chat.go @@ -0,0 +1,102 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type ChatHandler struct { + chatService *service.ChatService +} + +func NewChatHandler(chatService *service.ChatService) *ChatHandler { + return &ChatHandler{chatService: chatService} +} + +func (h *ChatHandler) CreateChat(w http.ResponseWriter, r *http.Request) { + + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + req, code := httputils.Decode[chatCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + result, err := h.chatService.Create(r.Context(), req.Title, req.MemberIDs, userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := chatResponse{ + Title: result.Title, + MemberIDs: result.MemberIDs, + ID: result.ID, + CreatedAt: httputils.FormatTime(result.CreatedAt), + } + + _ = httputils.WriteJSON[chatResponse](w, http.StatusCreated, response) + +} + +func (h *ChatHandler) GetChatsByUserID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + userID := middleware.UserIDFromContext(r.Context()) + + limit, cursor, ok := httputils.GetLimitAndCursor(r) + + if !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + chats, nextCursor, err := h.chatService.GetChatsByUserID(r.Context(), userID, limit, cursor) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + chatResponses := make([]chatResponse, len(chats)) + for index, chat := range chats { + chatResponses[index] = *convertChatModelToResponse(chat) + } + + response := chatGetByUserIDResponse{ + Items: chatResponses, + NextCursor: nextCursor, + } + + _ = httputils.WriteJSON[chatGetByUserIDResponse](w, http.StatusOK, response) + +} + +func (h *ChatHandler) GetChatByChatID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + userID := middleware.UserIDFromContext(r.Context()) + + chatID := r.PathValue("chat_id") + + chat, err := h.chatService.GetChatByID(r.Context(), chatID, userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertChatModelToResponse(chat) + _ = httputils.WriteJSON[chatResponse](w, http.StatusOK, *response) + +} diff --git a/internal/transport/http/handler/chat_types.go b/internal/transport/http/handler/chat_types.go new file mode 100644 index 00000000..0580ee11 --- /dev/null +++ b/internal/transport/http/handler/chat_types.go @@ -0,0 +1,32 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type chatCreateRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type chatResponse struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + ID string `json:"id"` + CreatedAt string `json:"created_at"` +} + +type chatGetByUserIDResponse struct { + Items []chatResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +func convertChatModelToResponse(model *model.Chat) *chatResponse { + return &chatResponse{ + Title: model.Title, + ID: model.ID, + MemberIDs: model.MemberIDs, + CreatedAt: httputils.FormatTime(model.CreatedAt), + } +} diff --git a/internal/transport/http/handler/health.go b/internal/transport/http/handler/health.go new file mode 100644 index 00000000..85aa5c19 --- /dev/null +++ b/internal/transport/http/handler/health.go @@ -0,0 +1,86 @@ +package handler + +import ( + "context" + "highload/internal/storage/s3" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "log/slog" + "net/http" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +type HealthHandler struct { + pool *pgxpool.Pool + redisClient *redis.Client + s3Storage *s3.Storage +} + +func NewHealthHandler(pool *pgxpool.Pool, redisClient *redis.Client, s3Storage *s3.Storage) *HealthHandler { + return &HealthHandler{pool: pool, redisClient: redisClient, s3Storage: s3Storage} +} + +func (h *HealthHandler) Live(w http.ResponseWriter, r *http.Request) { + response := HealthResponse{Status: "ok"} + err := httputils.WriteJSON[HealthResponse](w, http.StatusOK, response) + if err != nil { + slog.Error("failed to write health response", "error", err) + } +} + +func (h *HealthHandler) Ready(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + requestID := middleware.RequestIDFromContext(r.Context()) + if err := h.pool.Ping(ctx); err != nil { + slog.Error("database readiness check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + if err := h.redisClient.Ping(ctx).Err(); err != nil { + slog.Error("redis readiness check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + if err := h.s3Storage.Check(ctx); err != nil { + slog.Error("s3 storage check failed", "error", err) + if writeErr := httputils.WriteServiceUnavailable(w, requestID); writeErr != nil { + slog.Error("failed to write readiness response", "error", writeErr) + } + return + } + + response := HealthResponse{Status: "ok"} + err := httputils.WriteJSON[HealthResponse](w, http.StatusOK, response) + if err != nil { + slog.Error("failed to write health response", "error", err) + } + +} + +func (h *HealthHandler) NotFoundRoute(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + err := httputils.WriteError( + w, + http.StatusNotFound, + httputils.ErrorDetails{ + Code: "not_found", + Message: "Not Found", + RequestID: requestID, + }, + ) + if err != nil { + slog.Error("failed to write not found response", "error", err) + } +} diff --git a/internal/transport/http/handler/health_types.go b/internal/transport/http/handler/health_types.go new file mode 100644 index 00000000..e0b38e89 --- /dev/null +++ b/internal/transport/http/handler/health_types.go @@ -0,0 +1,5 @@ +package handler + +type HealthResponse struct { + Status string `json:"status"` +} diff --git a/internal/transport/http/handler/message.go b/internal/transport/http/handler/message.go new file mode 100644 index 00000000..5574faa7 --- /dev/null +++ b/internal/transport/http/handler/message.go @@ -0,0 +1,90 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type MessageHandler struct { + messageService *service.MessageService +} + +func NewMessageHandler(messageService *service.MessageService) *MessageHandler { + return &MessageHandler{messageService: messageService} +} + +func (h *MessageHandler) CreateMessage(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + chatID := r.PathValue("chat_id") + + req, code := httputils.Decode[messageCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + key := r.Header.Get("Idempotency-Key") + + if ok := httputils.CheckIdempotencyKeyPattern(key); !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{ + Code: "invalid_request", + Message: "wrong Idempotency-Key", + RequestID: requestID, + }) + return + } + + message, err := h.messageService.CreateMessage(r.Context(), req.Text, chatID, userID, key, req.AttachmentIDs) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertMessageModelToResponse(message) + + _ = httputils.WriteJSON[messageResponse](w, http.StatusCreated, *response) + +} + +func (h *MessageHandler) GetMessagesByChatID(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + chatID := r.PathValue("chat_id") + + limit, cursor, ok := httputils.GetLimitAndCursor(r) + + if !ok { + httputils.WriteError(w, http.StatusBadRequest, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + messages, nextCursor, err := h.messageService.GetMessagesByChatID(r.Context(), chatID, userID, limit, cursor) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + messageResponses := make([]messageResponse, len(messages)) + for index, message := range messages { + messageResponses[index] = *convertMessageModelToResponse(message) + } + + response := messageGetResponse{ + Items: messageResponses, + NextCursor: nextCursor, + } + + _ = httputils.WriteJSON[messageGetResponse](w, http.StatusOK, response) + +} diff --git a/internal/transport/http/handler/message_types.go b/internal/transport/http/handler/message_types.go new file mode 100644 index 00000000..09a431ab --- /dev/null +++ b/internal/transport/http/handler/message_types.go @@ -0,0 +1,42 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type messageCreateRequest struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type messageResponse struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type messageGetResponse struct { + Items []messageResponse `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +func convertMessageModelToResponse(message *model.Message) *messageResponse { + attachmentIDs := message.AttachmentIDs + + if attachmentIDs == nil { + attachmentIDs = []string{} + } + + return &messageResponse{ + ID: message.ID, + ChatID: message.ChatID, + SenderID: message.SenderID, + Text: message.Text, + AttachmentIDs: attachmentIDs, + CreatedAt: httputils.FormatTime(message.CreatedAt), + } +} diff --git a/internal/transport/http/handler/user.go b/internal/transport/http/handler/user.go new file mode 100644 index 00000000..517a48e7 --- /dev/null +++ b/internal/transport/http/handler/user.go @@ -0,0 +1,60 @@ +package handler + +import ( + "highload/internal/service" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + "net/http" +) + +type UserHandler struct { + userService *service.UserService +} + +func NewUserHandler(service *service.UserService) *UserHandler { + return &UserHandler{userService: service} +} + +func (h *UserHandler) CreateUser(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + + req, code := httputils.Decode[userCreateRequest](r) + + if code != http.StatusOK { + if code == http.StatusUnsupportedMediaType { + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "unsupported_media_type", Message: "Неверный media type", RequestID: requestID}) + return + } + httputils.WriteError(w, code, httputils.ErrorDetails{Code: "invalid_request", Message: "Неверный запрос", RequestID: requestID}) + return + } + + user, err := h.userService.CreateUser(r.Context(), req.Username, req.DisplayName, req.Password) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertUserModelToResponse(user) + + _ = httputils.WriteJSON[userResponse](w, http.StatusCreated, *response) + +} + +func (h *UserHandler) GetCurrentUser(w http.ResponseWriter, r *http.Request) { + requestID := middleware.RequestIDFromContext(r.Context()) + userID := middleware.UserIDFromContext(r.Context()) + + user, err := h.userService.GetUser(r.Context(), userID) + + if err != nil { + httputils.WriteServiceError(w, err, requestID) + return + } + + response := convertUserModelToResponse(user) + + _ = httputils.WriteJSON[userResponse](w, http.StatusOK, *response) + +} diff --git a/internal/transport/http/handler/user_types.go b/internal/transport/http/handler/user_types.go new file mode 100644 index 00000000..5a404e6d --- /dev/null +++ b/internal/transport/http/handler/user_types.go @@ -0,0 +1,28 @@ +package handler + +import ( + "highload/internal/model" + "highload/internal/transport/http/httputils" +) + +type userCreateRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type userResponse struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +func convertUserModelToResponse(user *model.User) *userResponse { + return &userResponse{ + ID: user.ID, + Username: user.Username, + DisplayName: user.DisplayName, + CreatedAt: httputils.FormatTime(user.CreatedAt), + } +} diff --git a/internal/transport/http/httputils/helper.go b/internal/transport/http/httputils/helper.go new file mode 100644 index 00000000..73ca6462 --- /dev/null +++ b/internal/transport/http/httputils/helper.go @@ -0,0 +1,207 @@ +package httputils + +import ( + "encoding/json/v2" + "errors" + "highload/internal/service" + "io" + "log/slog" + "mime" + "net/http" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type ErrorResponse struct { + Error ErrorDetails `json:"error"` +} + +type ErrorDetails struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +func Decode[T any](r *http.Request) (*T, int) { + contentType := r.Header.Get("Content-Type") + + mediaType, _, err := mime.ParseMediaType(contentType) + + if err != nil || mediaType != "application/json" { + return nil, http.StatusUnsupportedMediaType + } + + body, err := io.ReadAll(r.Body) + if err != nil { + return nil, http.StatusBadRequest + } + + if !utf8.Valid(body) { + return nil, http.StatusBadRequest + } + + req := new(T) + + err = json.Unmarshal(body, req, json.RejectUnknownMembers(true)) + if err != nil { + return nil, http.StatusBadRequest + } + return req, http.StatusOK + +} + +func GetLimitAndCursor(r *http.Request) (limit *int, cursor *string, ok bool) { + query := r.URL.Query() + + var currentLimit *int + + if query.Has("limit") { + limitStr := query.Get("limit") + + if limitStr == "" { + return nil, nil, false + } + limitInt, err := strconv.Atoi(limitStr) + + if err != nil { + return nil, nil, false + } + + currentLimit = &limitInt + + } + + var currentCursor *string + if query.Has("cursor") { + cursor := query.Get("cursor") + currentCursor = &cursor + } + + return currentLimit, currentCursor, true +} + +func WriteJSON[T any](w http.ResponseWriter, status int, response T) error { + data, err := json.Marshal(response) + if err != nil { + return err + } + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, err = w.Write(data) + return err +} + +func ErrorFromService(err error) (int, string) { + + switch { + case errors.Is(err, service.ErrInvalidInput): + return http.StatusBadRequest, "invalid_request" + + case errors.Is(err, service.ErrNotFound): + return http.StatusNotFound, "not_found" + + case errors.Is(err, service.ErrAlreadyExists): + return http.StatusConflict, "conflict" + case errors.Is(err, service.ErrUnauthorized): + return http.StatusUnauthorized, "unauthorized" + case errors.Is(err, service.ErrServiceUnavailable): + return http.StatusServiceUnavailable, "unavailable" + case errors.Is(err, service.ErrConflict): + return http.StatusConflict, "conflict" + case errors.Is(err, service.ErrPayloadTooLarge): + return http.StatusRequestEntityTooLarge, "payload_too_large" + case errors.Is(err, service.ErrUnsupportMediaType): + return http.StatusUnsupportedMediaType, "unsupported_media_type" + default: + return http.StatusInternalServerError, "internal_error" + } +} + +func WriteBasicUnauthorized(w http.ResponseWriter, requestID string) error { + w.Header().Set( + "WWW-Authenticate", + `Basic realm="messenger", charset="UTF-8"`, + ) + + return WriteError(w, http.StatusUnauthorized, ErrorDetails{ + Code: "unauthorized", + Message: "Unauthorized", + RequestID: requestID, + }) +} + +func WriteBearerUnauthorized(w http.ResponseWriter, requestID string) error { + w.Header().Set( + "WWW-Authenticate", + `Bearer realm="messenger"`, + ) + + return WriteError(w, http.StatusUnauthorized, ErrorDetails{ + Code: "unauthorized", + Message: "Unauthorized", + RequestID: requestID, + }) +} + +func WriteError(w http.ResponseWriter, code int, errorDetails ErrorDetails) error { + return WriteJSON(w, code, ErrorResponse{Error: errorDetails}) + +} + +func WriteServiceUnavailable(w http.ResponseWriter, requestID string) error { + return WriteError( + w, + http.StatusServiceUnavailable, + ErrorDetails{ + Message: "unavailable", + Code: "unavailable", + RequestID: requestID, + }, + ) +} + +func WriteServiceError(w http.ResponseWriter, err error, requestID string) error { + status, code := ErrorFromService(err) + + if status >= http.StatusInternalServerError { + slog.Error("request failed", + "error", err, + "request_id", requestID, + "status", status, + "code", code, + ) + } + + return WriteJSON(w, status, ErrorResponse{ + Error: ErrorDetails{ + Code: code, + Message: http.StatusText(status), + RequestID: requestID, + }, + }) +} + +func ExtractBearerToken(r *http.Request) (string, bool) { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || + !strings.EqualFold(scheme, "Bearer") || + token == "" || + strings.ContainsAny(token, " \t\r\n") { + + return "", false + } + return token, true +} + +func FormatTime(t time.Time) string { + return t.UTC().Format(time.RFC3339Nano) +} + +func CheckIdempotencyKeyPattern(key string) bool { + patternReg := regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + return patternReg.MatchString(key) +} diff --git a/internal/transport/http/middleware/auth.go b/internal/transport/http/middleware/auth.go new file mode 100644 index 00000000..69bbec7d --- /dev/null +++ b/internal/transport/http/middleware/auth.go @@ -0,0 +1,71 @@ +package middleware + +import ( + "errors" + "highload/internal/service" + "highload/internal/transport/http/httputils" + "net/http" + + "github.com/google/uuid" +) + +type Middleware struct { + authService *service.AuthService + instanceID string +} + +func NewMiddleware(authService *service.AuthService, instanceID string) *Middleware { + + return &Middleware{ + authService: authService, + instanceID: instanceID, + } + +} + +func (m *Middleware) Auth(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := RequestIDFromContext(r.Context()) + + token, ok := httputils.ExtractBearerToken(r) + + if !ok { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + + userID, err := m.authService.ValidateSession(r.Context(), token) + + if err != nil { + if errors.Is(err, service.ErrUnauthorized) { + httputils.WriteBearerUnauthorized(w, requestID) + return + } + httputils.WriteServiceError(w, err, requestID) + return + } + + ctx := WithUserID(r.Context(), userID) + next.ServeHTTP(w, r.WithContext(ctx)) + + }) +} + +func (m *Middleware) RequestID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestID := uuid.NewString() + + w.Header().Set("X-Request-Id", requestID) + + ctx := WithRequestID(r.Context(), requestID) + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func (m *Middleware) InstanceID(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", m.instanceID) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/transport/http/middleware/context.go b/internal/transport/http/middleware/context.go new file mode 100644 index 00000000..6419c4ac --- /dev/null +++ b/internal/transport/http/middleware/context.go @@ -0,0 +1,29 @@ +package middleware + +import "context" + +// UserID +type contextKey string + +const userIDKey contextKey = "userID" + +func WithUserID(ctx context.Context, userID string) context.Context { + return context.WithValue(ctx, userIDKey, userID) +} + +func UserIDFromContext(ctx context.Context) string { + userID, _ := ctx.Value(userIDKey).(string) + return userID +} + +// RequestIDKey +const requestIDKey contextKey = "requestID" + +func RequestIDFromContext(ctx context.Context) string { + requestID, _ := ctx.Value(requestIDKey).(string) + return requestID +} + +func WithRequestID(ctx context.Context, requestID string) context.Context { + return context.WithValue(ctx, requestIDKey, requestID) +} diff --git a/internal/transport/http/middleware/logging.go b/internal/transport/http/middleware/logging.go new file mode 100644 index 00000000..a3675566 --- /dev/null +++ b/internal/transport/http/middleware/logging.go @@ -0,0 +1,39 @@ +package middleware + +import ( + "log/slog" + "net/http" + "time" +) + +type responseWriter struct { + http.ResponseWriter + status int +} + +func (w *responseWriter) WriteHeader(status int) { + w.status = status + w.ResponseWriter.WriteHeader(status) +} + +func (m *Middleware) Logging(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + + rw := &responseWriter{ + ResponseWriter: w, + status: http.StatusOK, + } + + next.ServeHTTP(rw, r) + + slog.Info( + "http request", + "method", r.Method, + "path", r.URL.Path, + "status", rw.status, + "duration", time.Since(start), + "request_id", RequestIDFromContext(r.Context()), + ) + }) +} diff --git a/internal/transport/http/router.go b/internal/transport/http/router.go new file mode 100644 index 00000000..634a5acd --- /dev/null +++ b/internal/transport/http/router.go @@ -0,0 +1,57 @@ +package http + +import ( + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + "net/http" +) + +type Router struct { + middleware *middleware.Middleware + authHandler *handler.AuthHandler + healthHandler *handler.HealthHandler + chatHandler *handler.ChatHandler + userHandler *handler.UserHandler + messageHandler *handler.MessageHandler + attachmentHandler *handler.AttachmentHandler +} + +func NewRouter(middleware *middleware.Middleware, authHandler *handler.AuthHandler, healthHandler *handler.HealthHandler, chatHandler *handler.ChatHandler, userHandler *handler.UserHandler, messageHandler *handler.MessageHandler, attachmentHandler *handler.AttachmentHandler) *Router { + return &Router{chatHandler: chatHandler, authHandler: authHandler, healthHandler: healthHandler, middleware: middleware, userHandler: userHandler, messageHandler: messageHandler, attachmentHandler: attachmentHandler} +} + +func (r *Router) Handler() http.Handler { + mux := http.NewServeMux() + protectedMux := http.NewServeMux() + + protectedMux.HandleFunc("/", r.healthHandler.NotFoundRoute) + mux.HandleFunc("GET /health/live", r.healthHandler.Live) + mux.HandleFunc("GET /health/ready", r.healthHandler.Ready) + + //Auth + mux.HandleFunc("POST /api/v1/auth/sessions", r.authHandler.CreateSession) + protectedMux.HandleFunc("DELETE /api/v1/auth/sessions/current", r.authHandler.DeleteSession) + + //User + mux.HandleFunc("POST /api/v1/users", r.userHandler.CreateUser) + protectedMux.HandleFunc("GET /api/v1/users/me", r.userHandler.GetCurrentUser) + + //Chat + protectedMux.HandleFunc("POST /api/v1/chats", r.chatHandler.CreateChat) + protectedMux.HandleFunc("GET /api/v1/chats", r.chatHandler.GetChatsByUserID) + protectedMux.HandleFunc("GET /api/v1/chats/{chat_id}", r.chatHandler.GetChatByChatID) + + //Message + protectedMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", r.messageHandler.CreateMessage) + protectedMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", r.messageHandler.GetMessagesByChatID) + + //Attachment + protectedMux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", r.attachmentHandler.CreateAttachment) + protectedMux.HandleFunc("GET /api/v1/attachments/{attachment_id}", r.attachmentHandler.GetAttachment) + protectedMux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", r.attachmentHandler.GetAttachmentContent) + + mux.Handle("/api/v1/", r.middleware.Auth(protectedMux)) + return r.middleware.InstanceID(r.middleware.RequestID( + r.middleware.Logging(mux), + )) +} diff --git a/scripts/init-local.sh b/scripts/init-local.sh new file mode 100755 index 00000000..786cf3da --- /dev/null +++ b/scripts/init-local.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +umask 077 + +cd "$(dirname "$0")/.." +command -v openssl >/dev/null 2>&1 || { echo 'openssl is required' >&2; exit 1; } +mkdir -p secrets + +if [ ! -e secrets/postgres_password ] && [ ! -e secrets/database_url ]; then + password=$(openssl rand -hex 24) + printf '%s' "$password" > secrets/postgres_password + printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' "$password" > secrets/database_url +elif [ ! -s secrets/postgres_password ] || [ ! -s secrets/database_url ]; then + echo 'incomplete PostgreSQL secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e secrets/rustfs_root_access_key ] && [ ! -e secrets/rustfs_root_secret_key ]; then + openssl rand -hex 16 > secrets/rustfs_root_access_key + openssl rand -hex 32 > secrets/rustfs_root_secret_key +elif [ ! -s secrets/rustfs_root_access_key ] || [ ! -s secrets/rustfs_root_secret_key ]; then + echo 'incomplete RustFS root secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e secrets/s3_access_key_id ] && [ ! -e secrets/s3_secret_access_key ]; then + openssl rand -hex 16 > secrets/s3_access_key_id + openssl rand -hex 32 > secrets/s3_secret_access_key +elif [ ! -s secrets/s3_access_key_id ] || [ ! -s secrets/s3_secret_access_key ]; then + echo 'incomplete API S3 secrets; restore the matching pair' >&2 + exit 1 +fi + +if [ ! -e .env ]; then + cp .env.example .env +fi +echo 'Local .env and secret files are present. Existing values were kept.' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..e58c5fea --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,45 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." + +if [ -f .env ]; then + compose() { docker compose --env-file .env "$@"; } +else + : "${COMPOSE_PROJECT_NAME:?COMPOSE_PROJECT_NAME is required when .env is absent}" + compose() { docker compose "$@"; } +fi + +wait_healthy() { + for service in "$@"; do + containers=$(compose ps -q "$service") + [ -n "$containers" ] || { echo "$service has no running containers" >&2; return 1; } + for container in $containers; do + attempts=0 + until [ "$(docker inspect --format '{{.State.Health.Status}}' "$container")" = healthy ]; do + attempts=$((attempts + 1)) + [ "$attempts" -lt 30 ] || { echo "$service did not become healthy" >&2; return 1; } + sleep 1 + done + done + done +} + +restart_services() { + for service in "$@"; do + containers=$(compose ps -a -q "$service") + [ -n "$containers" ] || { echo "$service has no containers" >&2; return 1; } + docker restart --timeout 20 $containers >/dev/null + done +} + +# Restart only long-running services and preserve their containers and volumes. +cleanup=$(compose ps -a -q attachment-cleanup) +[ -n "$cleanup" ] || { echo 'attachment-cleanup has no container' >&2; exit 1; } +docker stop --timeout 20 "$cleanup" >/dev/null +restart_services postgres redis s3 +wait_healthy postgres redis s3 +restart_services api +wait_healthy api +restart_services lb +docker start "$cleanup" >/dev/null diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..deca2b10 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +cd "$(dirname "$0")/.." + +for container_id in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --timeout 5 "$container_id" >/dev/null + exit 0 + fi +done + +printf 'API replica %s not found\n' "$TARGET_INSTANCE_ID" >&2 +exit 1 \ No newline at end of file diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 00000000..65192b2d --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,11 @@ +version: "2" + +sql: + - engine: "postgresql" + schema: "db/migrations" + queries: "db/queries" + gen: + go: + package: "sqlc" + out: "internal/database/sqlc" + sql_package: "pgx/v5" \ No newline at end of file diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..30157663 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,65 @@ +# Собственные проверки HighLoad, лабораторная 4 + +S3-инфраструктура на `highload-lab4:8081`: [автоматические проверки и запуск](lab4/S3.md). +Они подключены к прежнему `attachment_bucket_security` hook; результаты +не подменяют проверку Go upload/download. + +Один актуальный набор: [lab4/run.py](lab4/run.py), общий запуск +[verify.py](verify.py), существовавшие собственные Go-проверки в `auth_tests/`. +Go-проверки перенесены из собственных тестов lab2; устаревшие конструкторы +обновлены до `NewMiddleware(auth, instanceID)` и +`NewMessageRepository(queries, pool)`. HTTP/HA runner переиспользован из lab3 +и дополнен под контракт lab4. Соседние репозитории при запуске не нужны. +Параллельного альтернативного набора для тех же сценариев нет. + +`tests/`, `contracts/`, приложение, миграции и рабочие настройки не меняются. +Тесты создают синтетические данные; restart/failover hooks останавливают +процессы выбранного учебного проекта. Volumes не удаляются. + +```sh +# Запускать из HighLoad на учебном стенде. Выбрать именно свой project. +export COMPOSE_PROJECT_NAME=highload-lab4 +python3 student-tests/verify.py --project "$COMPOSE_PROJECT_NAME" \ + --config student-tests/lab4/config.example.json \ + --output student-tests/results/run-01 +``` + +В конфигурации укажите **публичный адрес LB**, CA при HTTPS и hooks. +Каталог результатов должен быть новым/пустым. Общий запуск включает +`make check`, `go test -count=1 -json ./...`, проверки runner и все HTTP-сценарии, +включая реальные вызовы публичных `make test`, `make test-persistence` и +`make test-failover`. Для stand-alone Go достаточно `go test ./...`. +При ограничениях на системный кэш: `GOCACHE=/private/tmp/highload-lab4-go-cache go test ./...`. + +`PASS` означает выполненный конкретный сценарий. `FAIL` означает нарушение +проверки; `PENDING_INFRA` — нет стенда, адаптера или фактических lifecycle +доказательств. Ненулевой итоговый код возвращается и при FAIL, и при PENDING. +Go `SKIP` сохраняются поимённо в общем summary и отдельно помечаются +`PENDING_INFRA`: успешная сборка не доказывает проверку PostgreSQL/Redis/restart. + +Для существующих Go-интеграционных проверок задаются +`AUTH_TEST_DATABASE_URL`, `AUTH_TEST_REDIS_ADDR`, `AUTH_TEST_REDIS_RESTART_HOOK`, +`AUTH_TEST_BASE_URL` (тот же публичный LB), `AUTH_TEST_STACK_RESTART_HOOK` и +`MIGRATE_TEST_DATABASE_URL`. Используйте выделенные учебные stores; PostgreSQL +фикстуры создают/удаляют собственные уникальные schemas, Redis-проверки +перезапускают указанный Redis и работают с синтетическими ключами. DSN/пароли +передавайте в окружении локально, не в команде/отчёте/Git. Наличие настройки +не означает успешный тест: все assertions сохранены. Существующие проверки +диагностики, durability и временных бюджетов — наши дополнительные требования +качества там, где они строже обязательного контракта преподавателя. + +Для транзакции сообщения с вложениями до подключения S3 добавлены проверки +`TestMessageAttachments*` в том же Go-пакете. Они используют только +`LAB4_TEST_DATABASE_URL` и общую изолированную PostgreSQL-фикстуру; Redis/S3 +не нужны. Запуск и точное покрытие — в разделе «Транзакция Message + attachments» +[lab4/README.md](lab4/README.md). Без DSN — явный SKIP/PENDING, не PASS. +Исключение — сервисная `TestMessageAttachmentsSingleIDServiceRegression`: +она не требует внешних сервисов, ловит panic на одном вложении и проверяет +два вызова с теми же значениями и UUID заглавными буквами. Реальную SQL-idempotency +проверяют прежние PostgreSQL-группы, расширенные таким же неизменным JSON. + +Подробные сценарии, границы доказательства и интерфейс адаптеров: +[lab4/README.md](lab4/README.md). Итоговые результаты приведены в [REPORT.md](../REPORT.md). Прогон не выставляет оценку автоматически: +lab4 на «4» требует всех критериев её уровня 3 и уровня 4; для «5» дополнительно +нужен один завершённый исследовательский трек. Нагрузочные исследования на «5» +в этот набор не включены. diff --git a/student-tests/auth_contract.py b/student-tests/auth_contract.py new file mode 100644 index 00000000..3e0e513a --- /dev/null +++ b/student-tests/auth_contract.py @@ -0,0 +1,538 @@ +#!/usr/bin/env python3 +"""Lab2 auth acceptance tests. Uses the unchanged teacher HTTP client. + +No server startup, automatic Docker actions, retries of writes or secret output. +Infrastructure hooks run only in explicitly selected suites. +""" +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import json +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import unittest +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, LAB, check, timestamp # noqa: E402 + +LOGIN = "/api/v1/auth/sessions" +LOGOUT = LOGIN + "/current" +ME = "/api/v1/users/me" +ARGS = None + + +def basic(username, password): + value = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + value} + + +def registration(password=None): + return { + "username": "auth_" + uuid.uuid4().hex[:20], + "display_name": "Проверка auth", + "password": password if password is not None else "Aa:" + uuid.uuid4().hex, + } + + +def new_client(timeout=None): + return Client(ARGS.base_url, ARGS.ca_file, timeout=timeout or ARGS.timeout) + + +def error(c, method, path, status, code, **kwargs): + if LAB >= 3 and method == "POST" and path.endswith("/messages"): + kwargs["headers"] = {**kwargs.get("headers", {}), "Idempotency-Key": uuid.uuid4().hex} + response = c.call(method, path, status, **kwargs) + check(response["json"]["error"]["code"] == code, "incorrect contract error code") + if status == 401: + challenge = ('Basic realm="messenger", charset="UTF-8"' + if path == LOGIN else 'Bearer realm="messenger"') + check(response["headers"].get("WWW-Authenticate") == challenge, + "incorrect authentication challenge") + return response + + +def create_actor(c, password=None): + body = registration(password) + user = c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + session = c.call("POST", LOGIN, 201, + headers=basic(body["username"], body["password"]), schema="Session")["json"] + check(session["user"] == user, "login returned a different user") + check(timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc), + "new session is already expired") + return {"id": user["id"], "user": user, "registration": body, + "token": session["token"], "expires_at": session["expires_at"]} + + +def login(c, actor): + return c.call("POST", LOGIN, 201, schema="Session", + headers=basic(actor["registration"]["username"], + actor["registration"]["password"]))["json"] + + +def chat(c, owner, members=None): + members = members if members is not None else [owner["id"]] + return c.call("POST", "/api/v1/chats", 201, actor=owner, + body={"title": "Auth regression", "member_ids": members}, schema="Chat")["json"] + + +class AuthContract(unittest.TestCase): + """Independent cases: a logout or rejection never poisons another test.""" + + def setUp(self): + self.c = new_client() + + def test_public_health_and_registration(self): + for path in ("/health/live", "/health/ready"): + with self.subTest(path=path): + self.c.call("GET", path, schema="Health") + body = registration() + # Registration is public even when a stale credential is supplied. + user = self.c.call("POST", "/api/v1/users", 201, body=body, + headers={"Authorization": "Bearer invalid"}, schema="User")["json"] + check(user["username"] == body["username"], "username changed") + check(user["display_name"] == body["display_name"], "display_name changed") + + def test_password_unicode_boundaries_and_colon(self): + for label, password in ( + ("ascii_min", "a" * 12), ("ascii_max", "b" * 128), + ("unicode_min", "😀" * 12), ("unicode_max", "😀" * 128), + ("colon", "начало:пароль:конец"), + ("spaces_are_significant", " significant-password "), + ): + with self.subTest(case=label): + actor = create_actor(self.c, password) + me = self.c.call("GET", ME, actor=actor, schema="User")["json"] + check(me == actor["user"], "current user differs from registration") + altered = password.strip() if password != password.strip() else password + "x" + error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(actor["registration"]["username"], altered)) + + def test_invalid_password_length_and_blank(self): + for label, value in ( + ("empty", ""), ("ascii_short", "a" * 11), ("ascii_long", "a" * 129), + ("unicode_short", "😀" * 11), ("unicode_long", "😀" * 129), + ("blank_ascii", " " * 12), ("blank_unicode", "\u2003" * 12), + ): + with self.subTest(case=label): + body = registration(value) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_registration_required_fields_types_and_names(self): + for field in ("username", "display_name", "password"): + for label, value in (("missing", None), ("null", None), ("integer", 1), + ("boolean", True), ("array", []), ("object", {})): + with self.subTest(field=field, case=label): + body = registration() + if label == "missing": + body.pop(field) + else: + body[field] = value + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for alias in (field.upper(), field[0].upper() + field[1:]): + for extra in (False, True): + with self.subTest(field=field, alias=alias, extra=extra): + body = registration() + body[alias] = body[field] if extra else body.pop(field) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for root in (None, True, 1, "string", []): + with self.subTest(root_type=type(root).__name__): + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=json.dumps(root).encode(), headers={"Content-Type": "application/json"}) + body = registration() + body["sender_id"] = str(uuid.uuid4()) + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + + def test_registration_bytes_json_and_content_type(self): + for label, raw in ( + ("empty", b""), ("malformed", b'{"username":'), + ("second_json", json.dumps(registration()).encode() + b"{}"), + ("trailing_garbage", json.dumps(registration()).encode() + b" garbage"), + ): + with self.subTest(case=label): + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=raw, headers={"Content-Type": "application/json"}) + for field in ("username", "display_name", "password"): + for label, bad in (("ff", b"\xff"), ("overlong", b"\xc0\xaf"), + ("truncated", b"\xe2\x82"), ("surrogate", b"\xed\xa0\x80")): + with self.subTest(field=field, encoding=label): + body = registration() + body[field] = "BYTE_MARKER" + raw = json.dumps(body).encode().replace(b"BYTE_MARKER", b"prefix" + bad + b"suffix") + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", + raw=raw, headers={"Content-Type": "application/json"}) + for media in ("text/plain", "application/octet-stream"): + with self.subTest(media=media): + error(self.c, "POST", "/api/v1/users", 415, "unsupported_media_type", + raw=json.dumps(registration()).encode(), headers={"Content-Type": media}) + + def test_username_and_display_name_constraints(self): + for username in ("ab", "a" * 33, "ABC", "имя", "bad-name", "bad name", " bad"): + with self.subTest(case="username", length=len(username)): + body = registration() + body["username"] = username + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for name in ("", " \t\n", "\u00a0\u2003", "😀" * 101): + with self.subTest(case="display_name", length=len(name)): + body = registration() + body["display_name"] = name + error(self.c, "POST", "/api/v1/users", 400, "invalid_request", body=body) + for length in (1, 100): + with self.subTest(case="valid_display_name", length=length): + body = registration() + body["display_name"] = "😀" * length + user = self.c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["display_name"] == body["display_name"], "Unicode display name changed") + body = registration() + body["username"] = uuid.uuid4().hex + user = self.c.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"] + check(user["username"] == body["username"], "maximum-length username changed") + + def test_wrong_credentials_have_same_public_error(self): + actor = create_actor(self.c) + responses = [] + for label, username, password in ( + ("wrong_password", actor["registration"]["username"], "wrong-password"), + ("unknown_user", registration()["username"], actor["registration"]["password"]), + ): + with self.subTest(case=label): + responses.append(error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(username, password))) + if len(responses) == 2: + check(responses[0]["json"]["error"]["message"] == + responses[1]["json"]["error"]["message"], + "credentials errors reveal whether username exists") + self.c.call("GET", ME, actor=actor, schema="User") + + def test_login_rejects_missing_or_malformed_basic(self): + actor = create_actor(self.c) + encoded_no_colon = base64.b64encode(b"username-only").decode() + for label, headers in ( + ("absent", {}), ("legacy_id", {"X-User-Id": actor["id"]}), + ("bearer", {"Authorization": "Bearer " + actor["token"]}), + ("empty_basic", {"Authorization": "Basic"}), + ("invalid_base64", {"Authorization": "Basic !!!"}), + ("missing_separator", {"Authorization": "Basic " + encoded_no_colon}), + ("invalid_username_utf8", {"Authorization": "Basic " + base64.b64encode(b"\xffuser:synthetic-password").decode("ascii")}), + ("username_contains_nul", {"Authorization": "Basic " + base64.b64encode(b"bad\x00user:synthetic-password").decode("ascii")}), + ("wrong_scheme", {"Authorization": "Digest anything"}), + ("empty_password", basic(actor["registration"]["username"], "")), + ): + with self.subTest(case=label): + error(self.c, "POST", LOGIN, 401, "unauthorized", headers=headers) + + def test_login_does_not_accept_json_credentials(self): + actor = create_actor(self.c) + body = {"username": actor["registration"]["username"], + "password": actor["registration"]["password"]} + # Authentication precedes payload validation. + error(self.c, "POST", LOGIN, 401, "unauthorized", body=body) + error(self.c, "POST", LOGIN, 400, "invalid_request", body=body, + headers=basic(body["username"], body["password"])) + + def test_protected_routes_require_bearer_before_validation(self): + actor = create_actor(self.c) + basic_headers = basic(actor["registration"]["username"], actor["registration"]["password"]) + unknown = str(uuid.uuid4()) + routes = ( + ("GET", ME, None), ("GET", "/api/v1/chats", None), + ("POST", "/api/v1/chats", {"title": "", "member_ids": []}), + ("GET", "/api/v1/chats/" + unknown, None), + ("GET", "/api/v1/chats/not-a-uuid/messages?limit=0", None), + ("POST", "/api/v1/chats/" + unknown + "/messages", {"text": ""}), + ("DELETE", LOGOUT, None), + ) + for method, path, body in routes: + for label, headers in ( + ("absent", {}), ("basic", basic_headers), ("legacy_id", {"X-User-Id": actor["id"]}), + ("empty_bearer", {"Authorization": "Bearer"}), + ("unknown_token", {"Authorization": "Bearer " + uuid.uuid4().hex * 2}), + ("space_in_token", {"Authorization": "Bearer invalid token"}), + ): + with self.subTest(method=method, path=path, auth=label): + error(self.c, method, path, 401, "unauthorized", body=body, headers=headers) + + def test_identity_acl_and_sender_ignore_legacy_header(self): + alice, bob, eve = (create_actor(self.c) for _ in range(3)) + room = chat(self.c, alice, [alice["id"], bob["id"]]) + path = "/api/v1/chats/" + room["id"] + me = self.c.call("GET", ME, actor=alice, headers={"X-User-Id": eve["id"]}, + schema="User")["json"] + check(me == alice["user"], "legacy ID overrode session identity") + result = self.c.call("POST", path + "/messages", 201, actor=alice, + headers={"X-User-Id": bob["id"], + **({"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {})}, + body={"text": "Auth sender"}, schema="Message")["json"] + check(result["sender_id"] == alice["id"], "sender not derived from session") + for suffix in ("", "/messages"): + error(self.c, "GET", path + suffix, 404, "not_found", actor=eve, + headers={"X-User-Id": alice["id"]}) + error(self.c, "POST", path + "/messages", 404, "not_found", actor=eve, + body={"text": "Forbidden"}, headers={"X-User-Id": alice["id"]}) + hidden = self.c.call("GET", "/api/v1/chats?limit=100", actor=eve, + schema="ChatPage")["json"] + check(all(item["id"] != room["id"] for item in hidden["items"]), "foreign chat leaked") + page = self.c.call("GET", path + "/messages", actor=bob, schema="MessagePage")["json"] + check(page["items"] == [result], "confirmed message is not immediately visible") + error(self.c, "POST", path + "/messages", 400, "invalid_request", actor=alice, + body={"text": "Forged sender", "sender_id": bob["id"]}) + + def test_invalid_chat_does_not_leave_visible_partial_chat(self): + actor = create_actor(self.c) + before = self.c.call("GET", "/api/v1/chats?limit=100", actor=actor, schema="ChatPage")["json"] + for members in ([actor["id"], str(uuid.uuid4())], + [actor["id"], actor["id"]], + [actor["id"], actor["id"].upper()], [str(uuid.uuid4())]): + with self.subTest(member_count=len(members)): + error(self.c, "POST", "/api/v1/chats", 400, "invalid_request", actor=actor, + body={"title": "Must not exist", "member_ids": members}) + after = self.c.call("GET", "/api/v1/chats?limit=100", actor=actor, schema="ChatPage")["json"] + check(after == before, "rejected chat request left visible partial data") + + def test_logout_revokes_only_current_session(self): + actor = create_actor(self.c) + other = login(self.c, actor) + check(other["token"] != actor["token"], "separate logins reused a token") + self.c.call("DELETE", LOGOUT, 204, actor=actor) + error(self.c, "GET", ME, 401, "unauthorized", actor=actor) + error(self.c, "DELETE", LOGOUT, 401, "unauthorized", actor=actor) + second = dict(actor, token=other["token"]) + me = self.c.call("GET", ME, actor=second, schema="User")["json"] + check(me == actor["user"], "logout revoked another session") + fresh = login(self.c, actor) + check(fresh["token"] not in (actor["token"], other["token"]), "login reused an earlier token") + self.c.call("GET", ME, actor=dict(actor, token=fresh["token"]), schema="User") + + def test_session_ttl_matches_configuration(self): + actor = create_actor(self.c) + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(ARGS.ttl - 5 <= remaining <= ARGS.ttl + 5, "SESSION_TTL_SECONDS not respected") + for _ in range(3): + self.c.call("GET", ME, actor=actor, schema="User") + # Exact non-sliding deadline is checked by the storage and short-TTL suites. + + def test_tokens_distinct_opaque_and_tamper_rejected(self): + actor = create_actor(self.c) + issued = [actor["token"]] + [login(self.c, actor)["token"] for _ in range(3)] + check(len(set(issued)) == len(issued), "independent logins reused tokens") + for token in issued: + check(token != actor["id"] and token != actor["registration"]["username"], + "token contains identity instead of an opaque credential") + check(token != actor["registration"]["password"], "password used as session token") + token = actor["token"] + changed = token[:-1] + ("A" if token[-1] != "A" else "B") + error(self.c, "GET", ME, 401, "unauthorized", actor=dict(actor, token=changed)) + self.c.call("GET", ME, actor=actor, schema="User") + # Length/uniqueness is not proof of entropy. RNG is reviewed separately. + + def test_duplicate_username_concurrent_registration(self): + body = registration() + barrier = threading.Barrier(4) + + def send(index): + c = new_client() + candidate = dict(body, display_name="Race " + str(index), password=body["password"] + str(index)) + barrier.wait(timeout=ARGS.timeout) + result = c.call("POST", "/api/v1/users", (201, 409), body=candidate, schema="User") + if result["status"] == 409: + check(result["json"]["error"]["code"] == "conflict", "race loser did not receive conflict") + return candidate, result + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(send, range(4))) + winners = [(body, result) for body, result in results if result["status"] == 201] + check(len(winners) == 1, "username race must have exactly one winner") + winner_body, result = winners[0] + session = self.c.call("POST", LOGIN, 201, headers=basic(winner_body["username"], + winner_body["password"]), schema="Session")["json"] + check(session["user"] == result["json"], "race winner was overwritten") + for candidate, response in results: + if response["status"] == 409: + error(self.c, "POST", LOGIN, 401, "unauthorized", + headers=basic(candidate["username"], candidate["password"])) + + +def hook(path): + script = Path(path).expanduser().resolve() + check(script.is_file() and os.access(script, os.X_OK), "hook must exist and be executable") + check(".example." not in script.name, "implement a real hook first") + try: + result = subprocess.run([str(script)], stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=ARGS.hook_timeout) + except (OSError, subprocess.TimeoutExpired): + raise Failure("infrastructure hook failed or timed out; inspect it locally") from None + check(result.returncode == 0, "infrastructure hook returned an error") + + +def wait_ready(c): + deadline = time.monotonic() + ARGS.hook_timeout + while True: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "API readiness did not recover") + time.sleep(0.25) + + +def expiry(): + c = new_client() + actor = create_actor(c) + start = time.monotonic() + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(0 < remaining <= ARGS.ttl + 5, "short TTL not applied") + c.call("GET", ME, actor=actor, schema="User") + # Reading halfway must not renew the original deadline. + time.sleep(max(0, remaining / 2)) + c.call("GET", ME, actor=actor, schema="User") + time.sleep(max(0, remaining + 0.25 - (time.monotonic() - start))) + error(c, "GET", ME, 401, "unauthorized", actor=actor) + error(c, "DELETE", LOGOUT, 401, "unauthorized", actor=actor) + fresh = login(c, actor) + check(fresh["token"] != actor["token"], "expired token reused") + c.call("GET", ME, actor=dict(actor, token=fresh["token"]), schema="User") + + +def persistence(): + c = new_client(timeout=2) + actor = create_actor(c) + revoked = dict(actor, token=login(c, actor)["token"]) + c.call("DELETE", LOGOUT, 204, actor=revoked) + room = chat(c, actor) + original = c.message(actor, room, "Auth persistence")["json"] + remaining = (timestamp(actor["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(remaining > 2 * ARGS.hook_timeout + 10, "TTL too short for restart bounds") + hook(ARGS.restart_hook) + wait_ready(c) + # No re-login before checking the original token. + check(c.call("GET", ME, actor=actor, schema="User")["json"] == actor["user"], + "original session/user did not survive restart") + check(c.call("GET", "/api/v1/chats/" + room["id"], actor=actor, schema="Chat")["json"] == room, + "chat/membership changed after restart") + page = c.call("GET", "/api/v1/chats/" + room["id"] + "/messages", + actor=actor, schema="MessagePage")["json"] + check(page["items"] == [original], "confirmed message changed after restart") + error(c, "GET", ME, 401, "unauthorized", actor=revoked) + error(c, "DELETE", LOGOUT, 401, "unauthorized", actor=revoked) + c.call("DELETE", LOGOUT, 204, actor=actor) + error(c, "GET", ME, 401, "unauthorized", actor=actor) + check(login(c, actor)["user"] == actor["user"], "password/user not persistent") + # HTTP exposes expiry only at login. Storage test verifies deadline stability. + + +def dependency(): + c = new_client() + actor = create_actor(c) + failures = [] + fallback_session = None + revoked_during_outage = False + sql_with_redis = ARGS.dependency == "postgres" and getattr(ARGS, "session_store", "redis") == "redis" + if sql_with_redis: + fallback_session = dict(actor, token=login(c, actor)["token"]) + + def observe(name, operation): + try: + operation() + print("PASS:", name) + return True + except Failure as exc: + failures.append(name + ": " + str(exc)) + print("FAIL:", name + ": " + str(exc)) + return False + + try: + hook(ARGS.failure_hook) + observe("liveness during outage", lambda: c.call("GET", "/health/live", schema="Health")) + critical = not (ARGS.dependency == "redis" and ARGS.cache_policy == "fallback") + if critical: + observe("readiness during outage", lambda: error(c, "GET", "/health/ready", 503, "unavailable")) + observe("valid session during outage", lambda: error(c, "GET", ME, 503, "unavailable", actor=actor)) + else: + observe("fallback readiness", lambda: c.call("GET", "/health/ready", schema="Health")) + observe("fallback identity", lambda: c.call("GET", ME, actor=actor, schema="User")) + for name, headers in (("missing auth", {}), ("legacy identity", {"X-User-Id": actor["id"]})): + observe(name, lambda headers=headers: error(c, "GET", ME, 401, "unauthorized", headers=headers)) + # A syntactically valid Bearer cannot be declared unknown while the + # sole session store is unavailable. The lookup failed, not identity. + unknown_status = 503 if ARGS.dependency == "redis" and critical else 401 + observe("unknown bearer during outage", lambda: error(c, "GET", ME, unknown_status, + "unavailable" if unknown_status == 503 else "unauthorized", + headers={"Authorization": "Bearer " + uuid.uuid4().hex * 2})) + if critical: + observe("login during outage", lambda: error(c, "POST", LOGIN, 503, "unavailable", + headers=basic(actor["registration"]["username"], actor["registration"]["password"]))) + if not critical: + fallback_session = dict(actor, token=login(c, actor)["token"]) + if sql_with_redis or not critical: + revoked_during_outage = observe("Redis logout independent of SQL", lambda: c.call("DELETE", LOGOUT, 204, actor=actor)) + else: + observe("logout during outage", lambda: error(c, "DELETE", LOGOUT, 503, "unavailable", actor=actor)) + if ARGS.dependency == "postgres": + body = registration() + observe("registration during SQL outage", lambda: error(c, "POST", "/api/v1/users", 503, "unavailable", body=body)) + finally: + hook(ARGS.recovery_hook) + observe("readiness recovers", lambda: wait_ready(c)) + observe("liveness recovers", lambda: c.call("GET", "/health/live", schema="Health")) + survivor = fallback_session or actor + observe("independent session after recovery", lambda: c.call("GET", ME, actor=survivor, schema="User")) + if revoked_during_outage: + observe("logout remains revoked after recovery", lambda: error(c, "GET", ME, 401, "unauthorized", actor=actor)) + if ARGS.dependency == "postgres": + observe("failed registration left no user", lambda: c.call("POST", "/api/v1/users", 201, body=body, schema="User")) + check(not failures, "; ".join(failures)) + + +def main(): + global ARGS + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--base-url", default=None) + p.add_argument("--ca-file", default=None) + p.add_argument("--timeout", type=float, default=15) + p.add_argument("--ttl", type=int, default=3600, help="actual server SESSION_TTL_SECONDS") + p.add_argument("--suite", choices=("normal", "expiry", "persistence", "dependency"), default="normal") + p.add_argument("--restart-hook") + p.add_argument("--failure-hook") + p.add_argument("--recovery-hook") + p.add_argument("--dependency", choices=("postgres", "redis")) + p.add_argument("--cache-policy", choices=("critical", "fallback")) + p.add_argument("--session-store", choices=("postgres", "redis"), default="redis", + help="declared session backend; controls logout expectation during a SQL outage") + p.add_argument("--hook-timeout", type=float, default=90) + ARGS = p.parse_args() + if LAB not in (2, 3, 4): + p.error("this suite targets the lab2 auth contract retained in lab3 and lab4") + if ARGS.timeout <= 0 or ARGS.hook_timeout <= 0: + p.error("timeouts must be positive") + if ARGS.suite == "expiry": + if not 2 <= ARGS.ttl <= 10: + p.error("expiry needs a separately configured server with TTL 2..10") + elif ARGS.ttl < 300: + p.error("normal, restart and dependency suites require TTL >= 300") + if ARGS.suite == "persistence" and not ARGS.restart_hook: + p.error("persistence requires an explicit --restart-hook") + if ARGS.suite == "dependency": + if not (ARGS.failure_hook and ARGS.recovery_hook and ARGS.dependency): + p.error("dependency suite needs failure/recovery hooks and the dependency name") + if ARGS.dependency == "redis" and not ARGS.cache_policy: + p.error("declare whether Redis is critical or has a safe fallback") + if ARGS.suite == "normal": + result = unittest.TextTestRunner(verbosity=2).run( + unittest.defaultTestLoader.loadTestsFromTestCase(AuthContract)) + raise SystemExit(0 if result.wasSuccessful() else 1) + try: + {"expiry": expiry, "persistence": persistence, "dependency": dependency}[ARGS.suite]() + except (Failure, ValueError, OSError) as exc: + print("FAIL:", str(exc)) + raise SystemExit(1) + print("PASS:", ARGS.suite, "(hook scope and runtime settings need separate evidence)") + + +if __name__ == "__main__": + main() diff --git a/student-tests/auth_tests/attachment_cleanup_test.go b/student-tests/auth_tests/attachment_cleanup_test.go new file mode 100644 index 00000000..19166462 --- /dev/null +++ b/student-tests/auth_tests/attachment_cleanup_test.go @@ -0,0 +1,217 @@ +package auth_tests + +import ( + "context" + "errors" + "io" + "net/http" + "slices" + "strings" + "testing" + "testing/synctest" + "time" + + "highload/internal/model" + "highload/internal/repository" + "highload/internal/service" + "highload/internal/storage" + "highload/internal/storage/s3" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" +) + +type cleanupRepo struct { + repository.AttachmentRepository + referenced map[string]bool + busy string + existsErr error + checked []string + locked string +} + +type cleanupTx struct{ repo *cleanupRepo } + +func (tx cleanupTx) CreateAttachment(context.Context, repository.CreateAttachmentParams) (*model.Attachment, error) { + return nil, errors.New("unexpected INSERT") +} +func (tx cleanupTx) GetAttachmentByID(context.Context, string) (*model.Attachment, error) { + return nil, errors.New("unexpected attachment lookup") +} +func (tx cleanupTx) ExistsByObjectKey(ctx context.Context, key string) (bool, error) { + r := tx.repo + if r.locked != key { + return false, errors.New("EXISTS outside locked transaction") + } + r.checked = append(r.checked, key) + return r.referenced[key], r.existsErr +} +func (r *cleanupRepo) WithObjectKeyLock(ctx context.Context, key string, fn func(context.Context, repository.AttachmentTx) error) (bool, error) { + if key == r.busy { + return false, nil + } + r.locked = key + defer func() { r.locked = "" }() + return true, fn(ctx, cleanupTx{repo: r}) +} + +type cleanupStore struct { + pages map[string][]storage.ObjectInfo + next map[string]string + listed, deleted []string + repo *cleanupRepo + listErr, deleteErr error + waitList bool +} + +func (s *cleanupStore) ListPage(ctx context.Context, token *string, prefix string) ([]storage.ObjectInfo, *string, error) { + if prefix != "attachments/" { + return nil, nil, errors.New("unexpected prefix") + } + if _, ok := ctx.Deadline(); !ok { + return nil, nil, errors.New("LIST did not receive deadline") + } + if s.waitList { + <-ctx.Done() + return nil, nil, ctx.Err() + } + key := aws.ToString(token) + s.listed = append(s.listed, key) + if s.listErr != nil { + return nil, nil, s.listErr + } + if next, ok := s.next[key]; ok { + return s.pages[key], &next, nil + } + return s.pages[key], nil, nil +} +func (s *cleanupStore) Delete(_ context.Context, key string) error { + if s.repo.locked != key { + return errors.New("DELETE outside locked transaction") + } + s.deleted = append(s.deleted, key) + return s.deleteErr +} + +func TestAttachmentCleanupPagesAndCutoff(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + now := time.Now() + old := now.Add(-time.Minute - time.Nanosecond) + orphan := "attachments/orphan" + opts := &service.AttachmentCleanupOptions{GracePeriod: time.Minute, ListTimeout: time.Second} + repo := &cleanupRepo{referenced: map[string]bool{"attachments/referenced": true}, busy: "attachments/busy"} + objects := &cleanupStore{repo: repo, pages: map[string][]storage.ObjectInfo{ + "": {{Key: "attachments/equal", LastModified: now.Add(-time.Minute)}, + {Key: "attachments/fresh", LastModified: now.Add(-time.Second)}, + {Key: "attachments/future", LastModified: now.Add(time.Hour)}, + {Key: "attachments/zero"}, + {Key: "other/orphan", LastModified: old}, + {Key: "attachments/busy", LastModified: old}}, + "last": {{Key: "attachments/referenced", LastModified: old}, {Key: orphan, LastModified: old}}, + }, next: map[string]string{"": "empty", "empty": "last"}} + svc := service.NewAttachmentCleanupService(repo, objects, opts) + opts.DryRun = true + if err := svc.RunOnce(context.Background()); err != nil || len(objects.deleted) != 0 || + !slices.Equal(repo.checked, []string{"attachments/referenced", orphan}) { + t.Fatalf("dry-run must check old unlocked keys without deleting: err=%v checked=%v deletes=%v", err, repo.checked, objects.deleted) + } + opts.DryRun = false + objects.listed = nil + if err := svc.RunOnce(context.Background()); err != nil || !slices.Equal(objects.listed, []string{"", "empty", "last"}) || + !slices.Equal(objects.deleted, []string{orphan}) { + t.Fatalf("wrong cutoff or page walk: err=%v pages=%v deletes=%v", err, objects.listed, objects.deleted) + } + }) +} +func TestAttachmentCleanupFaults(t *testing.T) { + for _, name := range []string{"list", "exists", "delete", "list_timeout", "pass_timeout", "canceled"} { + t.Run(name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + repo := &cleanupRepo{} + objects := &cleanupStore{repo: repo, pages: map[string][]storage.ObjectInfo{"": {{Key: "attachments/old", LastModified: time.Now().Add(-time.Hour)}}}} + svc := service.NewAttachmentCleanupService(repo, objects, &service.AttachmentCleanupOptions{GracePeriod: time.Second, ListTimeout: time.Second}) + sentinel := errors.New("synthetic " + name + " error") + var want error = sentinel + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + switch name { + case "list": + objects.listErr = sentinel + case "exists": + repo.existsErr = sentinel + case "delete": + objects.deleteErr = sentinel + case "list_timeout": + objects.waitList = true + want = context.DeadlineExceeded + case "pass_timeout": + var stop context.CancelFunc + ctx, stop = context.WithTimeout(ctx, 100*time.Millisecond) + defer stop() + objects.waitList = true + want = context.DeadlineExceeded + case "canceled": + cancel() + want = context.Canceled + } + start := time.Now() + got := svc.RunOnce(ctx) + if !errors.Is(got, want) { + t.Fatalf("want %v, got %v", want, got) + } + if (name == "list_timeout" && time.Since(start) != time.Second) || + (name == "pass_timeout" && time.Since(start) != 100*time.Millisecond) { + t.Fatal("wrong cancellation budget") + } + if name != "delete" && len(objects.deleted) > 0 { + t.Fatal("DELETE ran after unsafe precondition") + } + if name == "delete" && len(objects.deleted) != 1 { + t.Fatal("DELETE fault not reached") + } + repo.existsErr, objects.listErr, objects.deleteErr = nil, nil, nil + objects.waitList = false + objects.deleted = nil + if err := svc.RunOnce(context.Background()); err != nil || !slices.Equal(objects.deleted, []string{"attachments/old"}) { + t.Fatalf("cleanup did not recover: %v %v", err, objects.deleted) + } + }) + }) + } +} + +type cleanupListTransport func(*http.Request) (*http.Response, error) + +func (fn cleanupListTransport) Do(req *http.Request) (*http.Response, error) { return fn(req) } + +func TestAttachmentCleanupS3InvalidNextToken(t *testing.T) { + for _, tc := range []struct { + name string + token *string + xml string + }{ + {"missing", nil, ""}, + {"empty", nil, ""}, + {"repeat", aws.String("same"), "same"}, + } { + t.Run(tc.name, func(t *testing.T) { + n := 0 + obj := &s3.Storage{Bucket: "test", Client: awss3.New(awss3.Options{ + Region: "us-east-1", UsePathStyle: true, BaseEndpoint: aws.String("http://cleanup.invalid"), + Credentials: aws.AnonymousCredentials{}, Retryer: aws.NopRetryer{}, + HTTPClient: cleanupListTransport(func(req *http.Request) (*http.Response, error) { + n++ + if req.URL.Query().Get("continuation-token") != aws.ToString(tc.token) { + return nil, errors.New("wrong request token") + } + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader( + "true" + tc.xml + ""))}, nil + }), + })} + _, _, err := obj.ListPage(context.Background(), tc.token, "attachments/") + if !errors.Is(err, storage.ErrStorage) || n != 1 { + t.Fatalf("invalid token was not rejected: %v, calls=%d", err, n) + } + }) + } +} diff --git a/student-tests/auth_tests/attachment_tx_test.go b/student-tests/auth_tests/attachment_tx_test.go new file mode 100644 index 00000000..c745d3eb --- /dev/null +++ b/student-tests/auth_tests/attachment_tx_test.go @@ -0,0 +1,318 @@ +package auth_tests + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + repo "highload/internal/repository" + "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage" + "highload/internal/storage/s3" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" +) + +func newAttachmentTxFixture(t *testing.T) (*fixture, repo.CreateAttachmentParams) { + t.Helper() + f := newDatabaseFixture(t, "LAB4_TEST_DATABASE_URL", 4) + owner := uuid.NewString() + if _, err := f.pool.Exec(f.ctx, "INSERT INTO users(id,username,display_name,password_hash) VALUES($1,'tx_fixture','TX fixture','unused')", owner); err != nil { + t.Fatal(err) + } + chat, err := postgres.NewChatRepository(f.queries, f.pool).CreateChat(f.ctx, "Attachment transaction", []string{owner}) + if err != nil { + t.Fatal(err) + } + return f, repo.CreateAttachmentParams{ + ChatID: chat.ID, OwnerID: owner, ObjectKey: "attachments/" + uuid.NewString(), + Filename: "fixture.png", ContentType: "image/png", SizeBytes: 1, Sha256: strings.Repeat("a", 64), + } +} + +func assertAttachmentCount(t *testing.T, f *fixture, want int) { + t.Helper() + var count int + if err := f.pool.QueryRow(f.ctx, "SELECT count(*) FROM attachments").Scan(&count); err != nil || count != want { + t.Fatalf("expected %d committed attachments, got %d (%v)", want, count, err) + } +} + +func TestAttachmentTxLockAndRollback(t *testing.T) { + f, params := newAttachmentTxFixture(t) + // Run the isolation check on the very connection doing the callback INSERT. + // A query through the pool would only establish the observer's isolation. + if _, err := f.pool.Exec(f.ctx, ` + CREATE FUNCTION assert_attachment_isolation() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'expected read committed during attachment INSERT'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER assert_attachment_isolation BEFORE INSERT ON attachments + FOR EACH ROW EXECUTE FUNCTION assert_attachment_isolation(); + `, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal(err) + } + attachments := postgres.NewAttachmentRepository(f.queries, f.pool) + key := params.ObjectKey + entered := make(chan struct{}) + release := make(chan struct{}) + type outcome struct { + locked bool + err error + } + done := make(chan outcome, 1) + holdCtx, cancelHold := context.WithCancel(f.ctx) + defer cancelHold() + go func() { + locked, err := attachments.WithObjectKeyLock(holdCtx, key, func(ctx context.Context, _ repo.AttachmentTx) error { + close(entered) + select { + case <-release: + return nil + case <-ctx.Done(): + return ctx.Err() + } + }) + done <- outcome{locked, err} + }() + select { + case <-entered: + case result := <-done: + t.Fatalf("holder never entered callback: %+v", result) + case <-time.After(3 * time.Second): + t.Fatal("holder did not acquire the lock") + } + unexpected := func(context.Context, repo.AttachmentTx) error { + return errors.New("contending callback ran") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key, unexpected); locked || err != nil { + t.Fatalf("same key must return false, nil without callback: %t, %v", locked, err) + } + noop := func(context.Context, repo.AttachmentTx) error { return nil } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key+"-other", noop); !locked || err != nil { + t.Fatalf("another key was blocked: %t, %v", locked, err) + } + close(release) + select { + case result := <-done: + if !result.locked || result.err != nil { + t.Fatalf("holder did not complete: %+v", result) + } + case <-time.After(3 * time.Second): + t.Fatal("holder did not close its transaction") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, key, noop); !locked || err != nil { + t.Fatalf("committed lock was not released: %t, %v", locked, err) + } + + for _, name := range []string{"callback_error", "context_canceled"} { + t.Run(name, func(t *testing.T) { + sentinel := errors.New("synthetic callback failure") + ctx, cancel := context.WithCancel(f.ctx) + defer cancel() + want := error(sentinel) + if name == "context_canceled" { + want = context.Canceled + } + locked, err := attachments.WithObjectKeyLock(ctx, key, func(txCtx context.Context, tx repo.AttachmentTx) error { + if _, err := tx.CreateAttachment(txCtx, params); err != nil { + return err + } + exists, err := tx.ExistsByObjectKey(txCtx, key) + if err != nil || !exists { + return fmt.Errorf("transaction did not see its INSERT: %w", err) + } + assertAttachmentCount(t, f, 0) + if name == "context_canceled" { + cancel() + return txCtx.Err() + } + return sentinel + }) + if !locked || err != want { + t.Fatalf("callback error must pass through unchanged with true: %t, %v", locked, err) + } + assertAttachmentCount(t, f, 0) + if locked, err = attachments.WithObjectKeyLock(f.ctx, key, noop); !locked || err != nil { + t.Fatalf("rollback failed to free lock/pool: %t, %v", locked, err) + } + t.Log("INSERT ran at read committed, rollback left zero rows and released the lock") + }) + } +} + +// txS3Transport observes the real SDK requests and retains synthetic bytes for GET. +// Before accepting a PUT it challenges the lock from a separate PostgreSQL session. +type txS3Transport struct { + fixture *fixture + key string + data []byte + put, get, delete int + failPut bool +} + +func (s *txS3Transport) Do(request *http.Request) (*http.Response, error) { + resp := &http.Response{StatusCode: 200, Header: make(http.Header), Body: http.NoBody} + key := strings.TrimPrefix(request.URL.Path, "/tx-test/") + if !strings.HasPrefix(key, "attachments/") { + return nil, errors.New("unexpected synthetic S3 key") + } + switch request.Method { + case http.MethodPut: + s.put++ + s.key = key + var acquired bool + err := s.fixture.pool.QueryRow(request.Context(), + "SELECT pg_try_advisory_xact_lock(hashtextextended($1, 0))", key).Scan(&acquired) + if err != nil || acquired { + return nil, fmt.Errorf("PUT ran without held object lock: acquired=%t, error=%v", acquired, err) + } + if s.failPut { + resp.StatusCode = 503 + resp.Header.Set("Content-Type", "application/xml") + resp.Body = io.NopCloser(strings.NewReader("SlowDownSynthetic outage")) + return resp, nil + } + defer request.Body.Close() + s.data, err = io.ReadAll(request.Body) + if err != nil { + return nil, err + } + case http.MethodGet: + s.get++ + if key != s.key || s.data == nil { + return nil, errors.New("object unavailable in fixture") + } + resp.Body = io.NopCloser(bytes.NewReader(s.data)) + resp.ContentLength = int64(len(s.data)) + case http.MethodDelete: + s.delete++ + s.data = nil + default: + return nil, fmt.Errorf("unexpected S3 method %s", request.Method) + } + return resp, nil +} + +func TestAttachmentTxHTTP(t *testing.T) { + _, source, _, _ := runtime.Caller(0) + png, err := os.ReadFile(filepath.Join(filepath.Dir(source), "../../tests/fixtures/sample.png")) + if err != nil { + t.Fatal(err) + } + hash := fmt.Sprintf("%x", sha256.Sum256(png)) + for _, scenario := range []struct { + name string + faultSQL string + http int + errorCode string + }{ + {name: "s3_unavailable", http: 503, errorCode: "unavailable"}, + {name: "insert_after_put", faultSQL: "ALTER TABLE attachments ADD CHECK (filename <> 'fixture.png')", http: 500, errorCode: "internal_error"}, + {name: "commit_after_insert", faultSQL: ` + CREATE FUNCTION reject_attachment_commit() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN RAISE EXCEPTION 'synthetic deferred failure'; END $$; + CREATE CONSTRAINT TRIGGER reject_attachment_commit AFTER INSERT ON attachments + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION reject_attachment_commit(); + `, http: 500, errorCode: "internal_error"}, + {name: "committed_upload_and_get", http: 201}, + } { + t.Run(scenario.name, func(t *testing.T) { + f, params := newAttachmentTxFixture(t) + if scenario.faultSQL != "" { + if _, err := f.pool.Exec(f.ctx, scenario.faultSQL, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal(err) + } + } + transport := &txS3Transport{fixture: f, failPut: scenario.http == 503} + objects := &s3.Storage{Bucket: "tx-test", Client: awss3.New(awss3.Options{ + Region: "us-east-1", UsePathStyle: true, BaseEndpoint: aws.String("http://tx-s3.invalid"), + Credentials: aws.AnonymousCredentials{}, Retryer: aws.NopRetryer{}, HTTPClient: transport, + })} + chats := service.NewChatService(postgres.NewChatRepository(f.queries, f.pool), nil, nil) + attachments := postgres.NewAttachmentRepository(f.queries, f.pool) + svc := service.NewAttachmentService(attachments, chats, objects) + h := handler.NewAttachmentHandler(svc) + call := func(method, path, param, value string, data []byte, next http.HandlerFunc) *httptest.ResponseRecorder { + r := httptest.NewRequest(method, path, bytes.NewReader(data)) + r = r.WithContext(middleware.WithUserID(f.ctx, params.OwnerID)) + r.SetPathValue(param, value) + r.Header.Set("Content-Type", "image/png") + w := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "attachment-tx") + mid.InstanceID(mid.RequestID(next)).ServeHTTP(w, r) + return w + } + w := call(http.MethodPost, "/attachments?filename=fixture.png", "chat_id", params.ChatID, png, h.CreateAttachment) + if transport.put != 1 || transport.delete != 0 { + t.Fatalf("expected one PUT and zero DELETE, got %d/%d", transport.put, transport.delete) + } + if scenario.http != 201 { + assertContractError(t, w, scenario.http, scenario.errorCode) + assertAttachmentCount(t, f, 0) + if scenario.http == 503 { + if transport.data != nil { + t.Fatal("failed S3 PUT stored an object") + } + _, err := svc.CreateAttachment(f.ctx, params.ChatID, params.OwnerID, "fixture.png", "image/png", bytes.NewReader(png), int64(len(png))) + if !errors.Is(err, service.ErrServiceUnavailable) || errors.Is(err, storage.ErrStorage) { + t.Fatalf("storage category did not reach service as unavailable: %v", err) + } + } else if !bytes.Equal(transport.data, png) { + t.Fatal("failed database write did not leave the already stored original") + } + if locked, err := attachments.WithObjectKeyLock(f.ctx, transport.key, func(context.Context, repo.AttachmentTx) error { return nil }); !locked || err != nil { + t.Fatalf("failed upload left key locked: %t, %v", locked, err) + } + t.Logf("direct handler HTTP %d; committed rows=0; DELETE=0", w.Code) + return + } + if w.Code != 201 { + t.Fatalf("expected 201 after commit, got %d", w.Code) + } + assertAttachmentCount(t, f, 1) + var response struct { + ID string `json:"id"` + SHA string `json:"sha256"` + Status string `json:"status"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil || response.SHA != hash || response.Status != "ready" { + t.Fatal("invalid upload response or SHA-256") + } + row, err := attachments.GetAttachmentForUser(f.ctx, params.OwnerID, response.ID) + if err != nil || row.SHA256 != hash || row.ObjectKey != transport.key || row.SizeBytes != int64(len(png)) { + t.Fatal("committed row does not match uploaded file") + } + get := call(http.MethodGet, "/attachments/"+response.ID, "attachment_id", response.ID, nil, h.GetAttachment) + if get.Code != 200 || !bytes.Equal(bytes.TrimSpace(get.Body.Bytes()), bytes.TrimSpace(w.Body.Bytes())) { + t.Fatal("metadata GET changed") + } + body := call(http.MethodGet, "/attachments/"+response.ID+"/content", "attachment_id", response.ID, nil, h.GetAttachmentContent) + if body.Code != 200 || transport.get != 1 || !bytes.Equal(body.Body.Bytes(), png) || body.Header().Get("Content-Type") != "image/png" { + t.Fatal("content GET did not return the original") + } + t.Logf("HTTP 201, committed rows=1, GET metadata/content=200, SHA-256=%s", hash) + }) + } +} diff --git a/student-tests/auth_tests/auth_handler_test.go b/student-tests/auth_tests/auth_handler_test.go new file mode 100644 index 00000000..51b67fa7 --- /dev/null +++ b/student-tests/auth_tests/auth_handler_test.go @@ -0,0 +1,32 @@ +package auth_tests + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" +) + +func TestStorageAuthHandlerRejectsJSONBody(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + writes := f.redis.audit.writeCount() + auth := handler.NewAuthHandler(f.auth) + mid := middleware.NewMiddleware(f.auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + request.SetBasicAuth(user.Username, password) + request.Header.Set("Content-Type", "application/json") + recorder := httptest.NewRecorder() + // Exercise the real handler independently of Router wiring. + // This component test does not replace the end-to-end login acceptance test. + mid.RequestID(http.HandlerFunc(auth.CreateSession)).ServeHTTP(recorder, request) + if recorder.Code != http.StatusBadRequest { + t.Errorf("login with JSON body must return 400; received %d", recorder.Code) + } + if f.redis.audit.writeCount() != writes { + t.Error("login with forbidden JSON body attempted to create a session") + } +} diff --git a/student-tests/auth_tests/budgets_test.go b/student-tests/auth_tests/budgets_test.go new file mode 100644 index 00000000..1818c906 --- /dev/null +++ b/student-tests/auth_tests/budgets_test.go @@ -0,0 +1,209 @@ +package auth_tests + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strconv" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "highload/internal/database" + "highload/internal/database/sqlc" + "highload/internal/repository" + "highload/internal/repository/postgres" +) + +func operationBudget(t *testing.T) time.Duration { + t.Helper() + value := os.Getenv("AUTH_TEST_OPERATION_BUDGET_MS") + if value == "" { + t.Log("budget_policy=provisional_3000ms; teacher has no numeric threshold; approval remains pending") + return 3 * time.Second + } + n, err := strconv.Atoi(value) + if err != nil || n < 500 || n > 10000 { + t.Fatal("AUTH_TEST_OPERATION_BUDGET_MS must be 500..10000") + } + return time.Duration(n) * time.Millisecond +} + +func applicationPool(t *testing.T, f *fixture) *pgxpool.Pool { + t.Helper() + u, err := url.Parse(os.Getenv("AUTH_TEST_DATABASE_URL")) + if err != nil { + t.Fatal("invalid test URL") + } + q := u.Query() + q.Set("search_path", f.config.ConnConfig.RuntimeParams["search_path"]) + u.RawQuery = q.Encode() + // Actual production constructor: never inherit fixture statement_timeout. + pool, err := database.NewPostgres(f.ctx, u.String()) + if err != nil { + t.Fatal("application pool unavailable") + } + f.pool.Close() + f.pool = pool + f.queries = sqlc.New(pool) + return pool +} + +func blockOperation(t *testing.T, f *fixture, mode string) func() { + t.Helper() + if mode == "blocked_query" { + tx, err := f.admin.Begin(f.ctx) + if err != nil { + t.Fatal("lock transaction failed") + } + table := pgx.Identifier{f.config.ConnConfig.RuntimeParams["search_path"], "users"}.Sanitize() + if _, err = tx.Exec(f.ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal("isolated table lock failed") + } + return func() { + ctx, c := context.WithTimeout(context.Background(), 2*time.Second) + defer c() + _ = tx.Rollback(ctx) + } + } + max := f.pool.Config().MaxConns + if max <= 0 || max > 128 { + t.Fatal("test refuses unbounded/oversized pool") + } + held := make([]*pgxpool.Conn, 0, max) + for range max { + conn, err := f.pool.Acquire(f.ctx) + if err != nil { + for _, h := range held { + h.Release() + } + t.Fatal("pool saturation failed") + } + held = append(held, conn) + } + return func() { + for _, h := range held { + h.Release() + } + } +} + +func TestPostgresOperationsHaveBoundedWaits(t *testing.T) { + budget := operationBudget(t) + for _, mode := range []string{"blocked_query", "exhausted_pool"} { + for _, scenario := range []string{"application_budget", "shorter_caller_deadline", "released_before_budget"} { + t.Run(mode+"/"+scenario, func(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + applicationPool(t, f) + release := blockOperation(t, f, mode) + released := false + defer func() { + if !released { + release() + } + }() + safety := budget + 2*time.Second + if scenario == "shorter_caller_deadline" { + safety = 150 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), safety) + defer cancel() + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil).WithContext(ctx) + request.Header.Set("Authorization", "Bearer "+session.Token) + recorder := httptest.NewRecorder() + done := make(chan struct{}) + started := time.Now() + go func() { defer close(done); redisRouter(f, f.redis.client).ServeHTTP(recorder, request) }() + if scenario == "released_before_budget" { + time.Sleep(100 * time.Millisecond) + release() + released = true + } + <-done + elapsed := time.Since(started) + t.Logf("mode=%s scenario=%s elapsed_ms=%.3f caller_deadline_ms=%d budget_ms=%d caller_canceled=%t", mode, scenario, float64(elapsed)/float64(time.Millisecond), safety.Milliseconds(), budget.Milliseconds(), ctx.Err() != nil) + switch scenario { + case "application_budget": + if ctx.Err() != nil || elapsed > budget+250*time.Millisecond { + t.Error("application did not independently enforce its operation budget before test safety cancellation") + } + assertContractError(t, recorder, 503, "unavailable") + case "shorter_caller_deadline": + if elapsed > 750*time.Millisecond { + t.Error("short caller deadline was not honored") + } + assertContractError(t, recorder, 503, "unavailable") + case "released_before_budget": + if recorder.Code != 200 || elapsed >= budget { + t.Error("unblocked operation failed or missed the declared budget") + } + } + if !released { + release() + released = true + } + recovery := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/api/v1/users/me", nil) + req.Header.Set("Authorization", "Bearer "+session.Token) + recoveryCtx, c := context.WithTimeout(context.Background(), 2*time.Second) + defer c() + redisRouter(f, f.redis.client).ServeHTTP(recovery, req.WithContext(recoveryCtx)) + if recovery.Code != 200 { + t.Error("service did not recover after releasing lock/pool") + } + if f.pool.Stat().AcquiredConns() != 0 { + t.Error("application pool leaked an acquired connection") + } + }) + } + } +} + +func TestPostgresChatCancellationBeforeCommitIsAtomic(t *testing.T) { + f := newFixture(t) + user, _ := f.user(t) + applicationPool(t, f) + tx, err := f.admin.Begin(f.ctx) + if err != nil { + t.Fatal("cannot start membership blocker") + } + released := false + defer func() { + if !released { + _ = tx.Rollback(context.Background()) + } + }() + table := pgx.Identifier{f.config.ConnConfig.RuntimeParams["search_path"], "chat_members"}.Sanitize() + if _, err = tx.Exec(f.ctx, "LOCK TABLE "+table+" IN ACCESS EXCLUSIVE MODE"); err != nil { + t.Fatal("cannot block membership insertion") + } + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + repo := postgres.NewChatRepository(f.queries, f.pool) + _, err = repo.CreateChat(ctx, "Canceled before membership", []string{user.ID}) + if !errors.Is(err, repository.ErrServiceUnavailable) { + t.Errorf("want unavailable after controlled pre-commit cancellation, got category %v", err) + } + // The membership INSERT cannot finish while this lock is held, so COMMIT + // has not been reached. No ambiguous-commit inference is made here. + _ = tx.Rollback(context.Background()) + released = true + for _, name := range []string{"chats", "chat_members"} { + var count int + if f.pool.QueryRow(f.ctx, "SELECT count(*) FROM "+name).Scan(&count) != nil || count != 0 { + t.Error("pre-commit cancellation left partial chat data") + } + } + if _, err = repo.CreateChat(f.ctx, "Works after cancellation", []string{user.ID}); err != nil { + t.Error("chat repository failed to recover") + } + if f.pool.Stat().AcquiredConns() != 0 { + t.Error("chat cancellation leaked a connection") + } +} diff --git a/student-tests/auth_tests/errors_test.go b/student-tests/auth_tests/errors_test.go new file mode 100644 index 00000000..8bad516a --- /dev/null +++ b/student-tests/auth_tests/errors_test.go @@ -0,0 +1,273 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "highload/internal/database/sqlc" + repository "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/httputils" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgtype" +) + +func TestErrorDiagnosticsRecordedOnceAndPublicResponseIsSafe(t *testing.T) { + var output bytes.Buffer + original := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, nil))) + defer slog.SetDefault(original) + users := service.NewUserService(repository.NewUserRepository(sqlc.New(faultDB{cause: errors.New("synthetic-storage-diagnostic")}))) + _, cause := users.GetUser(context.Background(), uuid.NewString()) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", "Bearer synthetic-secret-not-for-logs") + recorder := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "auth-test") + mid.RequestID(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = httputils.WriteServiceError(w, cause, middleware.RequestIDFromContext(r.Context())) + })).ServeHTTP(recorder, request) + assertContractError(t, recorder, 500, "internal_error") + if strings.Contains(recorder.Body.String(), "synthetic-storage-diagnostic") { + t.Error("public response exposed internal cause") + } + if strings.Contains(output.String(), "synthetic-secret-not-for-logs") { + t.Error("server log exposed a Bearer credential") + } + var records []map[string]any + for _, line := range strings.Split(strings.TrimSpace(output.String()), "\n") { + if line == "" { + continue + } + var record map[string]any + if json.Unmarshal([]byte(line), &record) != nil { + t.Error("diagnostic log is not structured JSON") + continue + } + // Access records are not detailed duplicate error records. + if record["msg"] != "http request" { + records = append(records, record) + } + } + if len(records) != 1 { + t.Errorf("want exactly one detailed error at response boundary, got %d", len(records)) + return + } + record := records[0] + if _, malformed := record["!BADKEY"]; malformed { + t.Error("structured error log contains !BADKEY: slog arguments are not valid key/value pairs") + } + if record["request_id"] != recorder.Header().Get("X-Request-Id") || record["status"] != float64(500) || + (record["code"] != "internal_error" && record["error_code"] != "internal_error") { + t.Error("diagnostic is missing matching request_id/status/error code") + } + encoded, _ := json.Marshal(record) + if !bytes.Contains(encoded, []byte("synthetic-storage-diagnostic")) { + t.Error("diagnostic lost storage cause") + } +} + +// Faults enter through sqlc's database interface, then pass through the real +// repository, service and HTTP error mapper. Driver causes are deliberately +// not required to survive: the public category and response are the contract. +type errorCase struct { + name string + cause error + want error + status int + code string +} + +func databaseErrorCases() []errorCase { + cases := []errorCase{ + {"no_rows", pgx.ErrNoRows, service.ErrNotFound, 404, "not_found"}, + {"duplicate", &pgconn.PgError{Code: "23505"}, service.ErrAlreadyExists, 409, "conflict"}, + {"eof", io.EOF, service.ErrServiceUnavailable, 503, "unavailable"}, + {"deadline", context.DeadlineExceeded, service.ErrServiceUnavailable, 503, "unavailable"}, + {"unexpected_eof", io.ErrUnexpectedEOF, service.ErrServiceUnavailable, 503, "unavailable"}, + {"closed_connection", pgconn.ErrConnClosed, service.ErrServiceUnavailable, 503, "unavailable"}, + {"connection_refused", unavailableRow{}.Scan(), service.ErrServiceUnavailable, 503, "unavailable"}, + // Docker removes the stopped service from DNS. The real outage probe + // returned a ConnectError containing this error, without a net.OpError. + {"database_dns_missing", &net.DNSError{Err: "no such host", Name: "postgres", IsNotFound: true}, service.ErrServiceUnavailable, 503, "unavailable"}, + {"sql_syntax", &pgconn.PgError{Code: "42601", Message: "private SQL diagnostic"}, service.ErrServerError, 500, "internal_error"}, + {"unexpected", errors.New("private database diagnostic"), service.ErrServerError, 500, "internal_error"}, + } + for _, code := range []string{"53300", "57P03", "08006", "08001", "57P01", "57P02"} { + cases = append(cases, errorCase{"sqlstate_" + code, &pgconn.PgError{Code: code}, service.ErrServiceUnavailable, 503, "unavailable"}) + } + // Wrapping must not change how an existing error category is recognized. + originals := append([]errorCase(nil), cases...) + for _, tc := range originals { + tc.name = "wrapped_" + tc.name + tc.cause = fmt.Errorf("database operation: %w", tc.cause) + cases = append(cases, tc) + } + return cases +} + +type faultRow struct{ cause error } + +func (r faultRow) Scan(...any) error { return r.cause } + +type faultDB struct { + unavailableDB + cause error +} + +func (d faultDB) QueryRow(context.Context, string, ...any) pgx.Row { + return faultRow{d.cause} +} + +func assertContractError(t *testing.T, recorder *httptest.ResponseRecorder, status int, code string) { + t.Helper() + if recorder.Code != status { + t.Fatalf("want HTTP %d, got %d", status, recorder.Code) + } + if recorder.Header().Get("Content-Type") != "application/json" { + t.Fatal("error response must have application/json content type") + } + var payload struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` + } + decoder := json.NewDecoder(recorder.Body) + if err := decoder.Decode(&payload); err != nil { + t.Fatal("error response is not valid JSON") + } + if decoder.Decode(new(any)) != io.EOF { + t.Fatal("error response contains more than one JSON value") + } + if payload.Error.Code != code || payload.Error.RequestID == "" || + payload.Error.RequestID != recorder.Header().Get("X-Request-Id") { + t.Fatal("error category or request ID violates the API contract") + } + if payload.Error.Message == "" || strings.Contains(payload.Error.Message, "private") { + t.Fatal("error message is empty or discloses internal diagnostics") + } +} + +func TestErrorDatabaseCategoryReachesHTTP(t *testing.T) { + for _, tc := range databaseErrorCases() { + t.Run(tc.name, func(t *testing.T) { + users := service.NewUserService(repository.NewUserRepository(sqlc.New(faultDB{cause: tc.cause}))) + _, err := users.GetUser(context.Background(), uuid.NewString()) + if !errors.Is(err, tc.want) { + t.Fatalf("want category %v, got %v", tc.want, err) + } + if errors.Is(tc.cause, context.DeadlineExceeded) && !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) { + t.Error("deadline diagnostic context lost between repository and service") + } + recorder := httptest.NewRecorder() + mid := middleware.NewMiddleware(nil, "auth-test") + mid.RequestID(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = httputils.WriteServiceError(w, err, middleware.RequestIDFromContext(r.Context())) + })).ServeHTTP(recorder, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)) + assertContractError(t, recorder, tc.status, tc.code) + }) + } +} + +func TestErrorUnavailableSQLLoginKeepsCategory(t *testing.T) { + for _, tc := range databaseErrorCases() { + if tc.status != 503 { + continue + } + t.Run(tc.name, func(t *testing.T) { + queries := sqlc.New(faultDB{cause: tc.cause}) + auth := service.NewAuthService(repository.NewUserRepository(queries), nil, time.Hour) + mid := middleware.NewMiddleware(auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("synthetic_user", "Synthetic:password") + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(handler.NewAuthHandler(auth).CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 503, "unavailable") + }) + } +} + +// The caller exists; the other member lookup fails. This isolates the service +// branch that used to replace the received err with ErrServerError. +type memberFaultDB struct { + unavailableDB + caller uuid.UUID + cause error +} + +func (d memberFaultDB) QueryRow(_ context.Context, _ string, args ...any) pgx.Row { + id := uuid.UUID(args[0].(pgtype.UUID).Bytes) + if id == d.caller { + return memberUserRow{id} + } + return faultRow{d.cause} +} + +type memberUserRow struct{ id uuid.UUID } + +func (r memberUserRow) Scan(dst ...any) error { + *dst[0].(*pgtype.UUID) = pgtype.UUID{Bytes: [16]byte(r.id), Valid: true} + *dst[1].(*string) = "error_test_caller" + *dst[2].(*string) = "Error test caller" + *dst[3].(*string) = "unused password hash" + *dst[4].(*pgtype.Timestamptz) = pgtype.Timestamptz{Time: time.Unix(1, 0), Valid: true} + return nil +} + +func TestErrorChatMemberLookupKeepsOperationMeaning(t *testing.T) { + for _, tc := range databaseErrorCases() { + if tc.status != 503 && tc.status != 404 && tc.status != 500 { + continue + } + t.Run(tc.name, func(t *testing.T) { + caller := uuid.New() + queries := sqlc.New(memberFaultDB{caller: caller, cause: tc.cause}) + users := repository.NewUserRepository(queries) + // No writable repository: an invalid member list must never reach + // chat insertion. Accidentally continuing is a test failure. + chats := service.NewChatService(nil, users, service.NewUserService(users)) + _, err := chats.Create(context.Background(), "Error regression", []string{caller.String(), uuid.NewString()}, caller.String()) + want, status, code := tc.want, tc.status, tc.code + if tc.status == 404 { + // contracts/README.md:17: unknown members make POST /chats invalid. + want, status, code = service.ErrInvalidInput, 400, "invalid_request" + } + if !errors.Is(err, want) { + t.Fatalf("want category %v, got %v", want, err) + } + gotStatus, gotCode := httputils.ErrorFromService(err) + if gotStatus != status || gotCode != code { + t.Fatalf("want %d/%s, got %d/%s", status, code, gotStatus, gotCode) + } + }) + } +} + +func TestErrorMissingBasicPrecedesForbiddenBody(t *testing.T) { + mid := middleware.NewMiddleware(nil, "auth-test") + auth := handler.NewAuthHandler(nil) + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", strings.NewReader("{}")) + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(auth.CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 401, "unauthorized") + if recorder.Header().Get("WWW-Authenticate") != `Basic realm="messenger", charset="UTF-8"` { + t.Fatal("missing Basic challenge") + } +} diff --git a/student-tests/auth_tests/logout_diagnostic_test.go b/student-tests/auth_tests/logout_diagnostic_test.go new file mode 100644 index 00000000..33217a99 --- /dev/null +++ b/student-tests/auth_tests/logout_diagnostic_test.go @@ -0,0 +1,202 @@ +package auth_tests + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/google/uuid" + redisclient "github.com/redis/go-redis/v9" + "highload/internal/repository" +) + +type logoutObservationKey struct{} +type logoutObservation struct { + Group int `json:"clients"` + Round int `json:"round"` + PauseMS int `json:"pause_ms"` + ElapsedMS float64 `json:"elapsed_ms"` + WaitMS float64 `json:"waitaof_ms"` + DelID int64 `json:"del_connection_id"` + WaitID int64 `json:"wait_connection_id"` + Local *int64 `json:"local_fsynced"` + Replicas *int64 `json:"replicas_fsynced"` + Kind string `json:"kind"` + CommandError string `json:"command_error,omitempty"` + RepositoryError string `json:"repository_error,omitempty"` +} +type logoutAOFObservation struct{} + +func (h logoutAOFObservation) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (h logoutAOFObservation) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (h logoutAOFObservation) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + started := time.Now() + err := next(ctx, cmd) + row, _ := ctx.Value(logoutObservationKey{}).(*logoutObservation) + if row == nil { + return err + } + name := cmd.Name() + if name != "del" && name != "waitaof" { + return err + } + if name == "waitaof" { + row.WaitMS = float64(time.Since(started)) / float64(time.Millisecond) + if err != nil { + row.CommandError = err.Error() + row.Kind = "command_error" + } else { + if value, ok := cmd.(*redisclient.Cmd); ok { + if reply, ok := value.Val().([]any); ok && len(reply) == 2 { + local, lok := reply[0].(int64) + replicas, rok := reply[1].(int64) + if lok && rok { + row.Local = &local + row.Replicas = &replicas + row.Kind = "acknowledged" + if local != 1 { + row.Kind = "unacknowledged" + } + } + } + } + if row.Local == nil { + row.Kind = "unexpected_reply" + } + } + } + if err == nil { + // next is bound to this Conn's processing path. The observation adds a + // read-only CLIENT ID on that same path; it never gets a second pool slot. + id := redisclient.NewIntCmd(ctx, "CLIENT", "ID") + if e := next(ctx, id); e == nil { + if name == "del" { + row.DelID = id.Val() + } else { + row.WaitID = id.Val() + } + } + } + return err + } +} + +func TestRedisLogoutPersistenceDiagnostic(t *testing.T) { + f := newRedisFixture(t) + f.client.AddHook(logoutAOFObservation{}) + before := f.client.PoolStats() + rows := []logoutObservation{} + var mu sync.Mutex + stop := make(chan struct{}) + sampled := make(chan struct{}) + poolSamples := []map[string]any{} + go func() { + defer close(sampled) + ticker := time.NewTicker(50 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + poolSamples = append(poolSamples, map[string]any{"at": time.Now().UTC().Format(time.RFC3339Nano), "stats": f.client.PoolStats()}) + } + } + }() + defer func() { + close(stop) + <-sampled + deadline := time.Now().Add(time.Second) + for f.client.PoolStats().TotalConns != f.client.PoolStats().IdleConns && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + after := f.client.PoolStats() + if after.TotalConns != after.IdleConns || after.PendingRequests != 0 { + t.Error("logout left acquired/pending pool connections") + } + report := map[string]any{"pool_before": before, "pool_after": after, "pool_samples": poolSamples, "samples": rows, "production_options": map[string]any{"read_timeout_ms": f.client.Options().ReadTimeout.Milliseconds(), "pool_size": f.client.Options().PoolSize, "pool_timeout_ms": f.client.Options().PoolTimeout.Milliseconds()}, "instrumentation": "read-only CLIENT ID after DEL and WAITAOF, on the pinned processing path"} + if directory := os.Getenv("AUTH_TEST_DIAGNOSTICS_DIR"); directory != "" { + _ = os.MkdirAll(directory, 0700) + data, _ := json.MarshalIndent(report, "", " ") + if os.WriteFile(filepath.Join(directory, "redis-logout.json"), data, 0600) != nil { + t.Error("cannot save Redis diagnostics") + } + } + t.Logf("logout samples=%d pool_wait_count=%d wait_ns=%d pool_timeouts=%d final_busy=%d", len(rows), after.WaitCount-before.WaitCount, after.WaitDurationNs-before.WaitDurationNs, after.Timeouts-before.Timeouts, after.TotalConns-after.IdleConns) + }() + run := func(clients, round, pause int) { + var workers sync.WaitGroup + start := make(chan struct{}) + for range clients { + workers.Add(1) + go func() { + defer workers.Done() + // Two sessions for one synthetic identity: revoking one must leave the + // other valid, even while other workers revoke their own sessions. + first, other := f.trackSafe(uuid.NewString()), f.trackSafe(uuid.NewString()) + user := uuid.NewString() + if _, err := f.repo.Create(f.ctx, first[:], user, time.Minute); err != nil { + t.Error("create logout fixture failed") + return + } + if _, err := f.repo.Create(f.ctx, other[:], user, time.Minute); err != nil { + t.Error("create independent fixture failed") + return + } + <-start + time.Sleep(time.Duration(pause) * time.Millisecond) + row := logoutObservation{Group: clients, Round: round, PauseMS: pause} + ctx := context.WithValue(f.ctx, logoutObservationKey{}, &row) + began := time.Now() + _, err := f.repo.RevokeSession(ctx, first[:]) + row.ElapsedMS = float64(time.Since(began)) / float64(time.Millisecond) + if err != nil { + row.RepositoryError = err.Error() + t.Errorf("logout clients=%d kind=%s category=%v", clients, row.Kind, err) + } + if row.Kind == "unacknowledged" && !errors.Is(err, repository.ErrServiceUnavailable) { + t.Error("unacknowledged local fsync must preserve unavailable category") + } + if row.DelID == 0 || row.DelID != row.WaitID { + t.Error("DEL/WAITAOF connection identity not preserved") + } + if _, err = f.repo.GetSessionByTokenHash(f.ctx, other[:]); err != nil { + t.Error("concurrent logout affected independent session") + } + if _, err = f.repo.GetSessionByTokenHash(f.ctx, first[:]); !errors.Is(err, repository.ErrNotFound) { + t.Error("revoked session still available") + } + mu.Lock() + rows = append(rows, row) + mu.Unlock() + }() + } + close(start) + workers.Wait() + } + for i, pause := range []int{0, 137, 359, 613, 887, 1109} { + run(1, i+1, pause) + } + for _, clients := range []int{5, 10} { + for round := 1; round <= 3; round++ { + run(clients, round, 137*round) + } + } +} + +// track modifies fixture cleanup bookkeeping; concurrent probes synchronize it. +var diagnosticTrackMu sync.Mutex + +func (f *redisFixture) trackSafe(token string) [32]byte { + diagnosticTrackMu.Lock() + defer diagnosticTrackMu.Unlock() + return f.track(token) +} diff --git a/student-tests/auth_tests/message_attachments_test.go b/student-tests/auth_tests/message_attachments_test.go new file mode 100644 index 00000000..6fc22c5a --- /dev/null +++ b/student-tests/auth_tests/message_attachments_test.go @@ -0,0 +1,529 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "highload/internal/model" + domainrepo "highload/internal/repository" + postgresrepo "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" +) + +// Uses the real handler/service/PostgreSQL repository and metadata-only fixtures. +// Authentication is already covered by the existing auth tests: only the current +// user context is injected here. No S3 server, upload endpoint or Redis is needed. +type messageAttachmentFixture struct { + *fixture + http http.Handler + alice, bob string + chat, other *model.Chat + ids, foreign []string +} + +type attachmentMessage struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +// Small dependencies for the real service's one-attachment regression. They +// compare UUID identity without prescribing where normalization takes place. +// They do not implement persistence/idempotency: PostgreSQL cases prove that. +type singleAttachmentChatRepo struct { + domainrepo.ChatRepository + chat *model.Chat +} + +func (r *singleAttachmentChatRepo) GetChatByID(_ context.Context, id string) (*model.Chat, error) { + if id != r.chat.ID { + return nil, domainrepo.ErrNotFound + } + return r.chat, nil +} + +type singleAttachmentMessageRepo struct { + domainrepo.MessageRepository + message *model.Message + key string + calls [][]string +} + +func (r *singleAttachmentMessageRepo) CreateMessage(_ context.Context, chatID, userID, text, key string, ids []string) (*model.Message, error) { + r.calls = append(r.calls, append([]string{}, ids...)) + if chatID != r.message.ChatID || userID != r.message.SenderID || text != r.message.Text || key != r.key { + return nil, domainrepo.ErrServerError + } + if len(ids) != len(r.message.AttachmentIDs) { + return nil, domainrepo.ErrInvalidInput + } + for index, raw := range ids { + id, err := uuid.Parse(raw) + if err != nil || id.String() != r.message.AttachmentIDs[index] { + return nil, domainrepo.ErrInvalidInput + } + } + return r.message, nil +} + +func TestMessageAttachmentsSingleIDServiceRegression(t *testing.T) { + const attachmentID = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + for _, scenario := range []struct { + name, inputID string + requests int + }{ + {name: "single_attachment_without_panic", inputID: attachmentID, requests: 1}, + {name: "identical_uppercase_body_replay", inputID: strings.ToUpper(attachmentID), requests: 2}, + } { + t.Run(scenario.name, func(t *testing.T) { + defer func() { + if recovered := recover(); recovered != nil { + t.Errorf("CreateMessage with one valid attachment panicked: %v", recovered) + } + }() + userID, chatID, key := uuid.NewString(), uuid.NewString(), uuid.NewString() + original := &model.Message{ID: uuid.NewString(), ChatID: chatID, SenderID: userID, + Text: "", AttachmentIDs: []string{attachmentID}, CreatedAt: time.Unix(1700000000, 0).UTC()} + repo := &singleAttachmentMessageRepo{message: original, key: key} + chatRepo := &singleAttachmentChatRepo{chat: &model.Chat{ID: chatID, MemberIDs: []string{userID}}} + chats := service.NewChatService(chatRepo, nil, nil) + messages := service.NewMessageService(repo, chats, nil) + for range scenario.requests { + // The uppercase case starts uppercase and repeats EXACTLY the same + // values. It never assumes different string bodies must be equivalent. + input := []string{scenario.inputID} + message, err := messages.CreateMessage(context.Background(), "", chatID, userID, key, input) + if err != nil || !reflect.DeepEqual(message, original) { + t.Fatalf("one attachment / identical uppercase replay: expected original Message without conflict, got %v", err) + } + } + if len(repo.calls) == 0 { + t.Fatal("service did not create the message through its repository") + } + for _, ids := range repo.calls { + if len(ids) != 1 { + t.Fatal("service lost the single attachment") + } + id, err := uuid.Parse(ids[0]) + if err != nil || id.String() != attachmentID { + t.Fatal("service changed the attachment UUID identity") + } + } + }) + } +} + +func newMessageAttachmentFixture(t *testing.T) *messageAttachmentFixture { + t.Helper() + f := newDatabaseFixture(t, "LAB4_TEST_DATABASE_URL", 12) + result := &messageAttachmentFixture{fixture: f, alice: uuid.NewString(), bob: uuid.NewString()} + for _, id := range []string{result.alice, result.bob} { + _, err := f.pool.Exec(f.ctx, `INSERT INTO users(id,username,display_name,password_hash) + VALUES($1,$2,'Synthetic attachment test','fixture-only-not-used-for-login')`, + id, "attach_"+strings.ReplaceAll(id, "-", "")[:18]) + if err != nil { + t.Fatal("cannot prepare synthetic users") + } + } + chatRepo := postgresrepo.NewChatRepository(f.queries, f.pool) + var err error + result.chat, err = chatRepo.CreateChat(f.ctx, "Message attachments", []string{result.alice, result.bob}) + if err != nil { + t.Fatal("cannot prepare primary chat") + } + result.other, err = chatRepo.CreateChat(f.ctx, "Other attachment scope", []string{result.alice}) + if err != nil { + t.Fatal("cannot prepare other chat") + } + attachments := postgresrepo.NewAttachmentRepository(f.queries, f.pool) + for _, chat := range []*model.Chat{result.chat, result.other} { + for i := 0; i < 11; i++ { + att, err := attachments.CreateAttachment(f.ctx, domainrepo.CreateAttachmentParams{ + ChatID: chat.ID, OwnerID: result.alice, Filename: fmt.Sprintf("synthetic-%d.png", i), + ObjectKey: "metadata-only/" + uuid.NewString(), ContentType: "image/png", + Sha256: strings.Repeat("a", 64), SizeBytes: 1, + }) + if err != nil { + t.Fatal("cannot seed attachment metadata through its PostgreSQL repository") + } + if chat.ID == result.chat.ID { + result.ids = append(result.ids, att.ID) + } else { + result.foreign = append(result.foreign, att.ID) + } + } + } + // Deliberately disagree with UUID ordering to catch accidental ORDER BY id. + slices.Sort(result.ids) + slices.Reverse(result.ids) + users := service.NewUserService(postgresrepo.NewUserRepository(f.queries)) + chats := service.NewChatService(chatRepo, postgresrepo.NewUserRepository(f.queries), users) + messages := service.NewMessageService(postgresrepo.NewMessageRepository(f.queries, f.pool), chats, users) + h := handler.NewMessageHandler(messages) + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.GetMessagesByChatID) + mid := middleware.NewMiddleware(nil, "message-pg-test") + result.http = mid.InstanceID(mid.RequestID(mux)) + return result +} + +func (f *messageAttachmentFixture) call(actor, method, chatID, key string, body any) *httptest.ResponseRecorder { + var raw []byte + if body != nil { + raw, _ = json.Marshal(body) + } + request := httptest.NewRequest(method, "/api/v1/chats/"+chatID+"/messages?limit=100", bytes.NewReader(raw)) + request = request.WithContext(middleware.WithUserID(f.ctx, actor)) + request.Header.Set("Content-Type", "application/json") + if key != "" { + request.Header.Set("Idempotency-Key", key) + } + recorder := httptest.NewRecorder() + f.http.ServeHTTP(recorder, request) + return recorder +} + +func attachmentBody(text string, ids []string) map[string]any { + return map[string]any{"text": text, "attachment_ids": ids} +} + +func decodeAttachmentMessage(t *testing.T, response *httptest.ResponseRecorder) attachmentMessage { + t.Helper() + if response.Code != http.StatusCreated { + t.Fatalf("POST message: expected HTTP 201, got %d", response.Code) + } + var value map[string]json.RawMessage + if err := json.Unmarshal(response.Body.Bytes(), &value); err != nil || len(value) != 6 { + t.Fatal("Message must contain exactly the six lab4 contract fields, including attachment_ids") + } + var message attachmentMessage + if err := json.Unmarshal(response.Body.Bytes(), &message); err != nil || message.AttachmentIDs == nil { + t.Fatal("attachment_ids must be an array, including [] for a text-only message") + } + for _, id := range []string{message.ID, message.ChatID, message.SenderID} { + if _, err := uuid.Parse(id); err != nil { + t.Fatal("Message contains an invalid UUID") + } + } + if _, err := time.Parse(time.RFC3339Nano, message.CreatedAt); err != nil || !strings.HasSuffix(message.CreatedAt, "Z") { + t.Fatal("Message must preserve a server UTC timestamp") + } + if response.Header().Get("X-Request-Id") == "" || response.Header().Get("X-Instance-Id") != "message-pg-test" { + t.Fatal("message response is missing request/instance identity") + } + return message +} + +func (f *messageAttachmentFixture) history(t *testing.T) []attachmentMessage { + t.Helper() + r := f.call(f.bob, "GET", f.chat.ID, "", nil) + var page struct { + Items []attachmentMessage `json:"items"` + NextCursor *string `json:"next_cursor"` + } + if r.Code != 200 || json.Unmarshal(r.Body.Bytes(), &page) != nil || page.NextCursor != nil { + t.Fatal("cannot read the complete synthetic history") + } + for _, message := range page.Items { + if message.AttachmentIDs == nil { + t.Fatal("history must include attachment_ids arrays") + } + } + return page.Items +} + +func (f *messageAttachmentFixture) assertLinks(t *testing.T, message attachmentMessage, expected []string) { + t.Helper() + if !sameOrderedAttachmentUUIDs(message.AttachmentIDs, expected) { + t.Fatal("Message attachment order or contents differ from request") + } + rows, err := f.pool.Query(f.ctx, `SELECT attachment_id::text, position, chat_id::text + FROM messages_attachments WHERE message_id=$1 ORDER BY position`, message.ID) + if err != nil { + t.Fatal("cannot inspect attachment links") + } + defer rows.Close() + var actual []string + for rows.Next() { + var id, chatID string + var position int + if rows.Scan(&id, &position, &chatID) != nil || position != len(actual) || chatID != f.chat.ID { + t.Fatal("attachment link has incorrect position or chat scope") + } + actual = append(actual, id) + } + if rows.Err() != nil || !slices.Equal(actual, expected) { + t.Fatal("database links do not match exactly the submitted attachment order") + } +} + +func sameOrderedAttachmentUUIDs(actual, expected []string) bool { + if actual == nil || len(actual) != len(expected) { + return false + } + for index, raw := range actual { + id, err := uuid.Parse(raw) + want, wantErr := uuid.Parse(expected[index]) + if err != nil || wantErr != nil || id != want { + return false + } + } + return true +} + +func sameHistoryMessage(actual, expected attachmentMessage) bool { + if !sameOrderedAttachmentUUIDs(actual.AttachmentIDs, expected.AttachmentIDs) { + return false + } + actual.AttachmentIDs = expected.AttachmentIDs + return reflect.DeepEqual(actual, expected) +} + +func (f *messageAttachmentFixture) snapshot(t *testing.T) map[string][]string { + t.Helper() + state := make(map[string][]string) + for _, table := range []string{"messages", "messages_attachments", "message_idempotency", "attachments"} { + // Table names are fixed test constants, never request input. + rows, err := f.pool.Query(f.ctx, "SELECT to_jsonb(item)::text FROM "+table+" AS item ORDER BY to_jsonb(item)::text") + if err != nil { + t.Fatal("cannot inspect transaction state") + } + for rows.Next() { + var row string + if rows.Scan(&row) != nil { + t.Fatal("cannot decode transaction state") + } + state[table] = append(state[table], row) + } + err = rows.Err() + rows.Close() + if err != nil { + t.Fatal("cannot finish inspecting transaction state") + } + } + return state +} + +func (f *messageAttachmentFixture) assertUnchanged(t *testing.T, before map[string][]string) { + t.Helper() + if !reflect.DeepEqual(before, f.snapshot(t)) { + t.Fatal("failed/conflicting request changed messages, links, idempotency or metadata") + } +} + +func TestMessageAttachmentsTextCompatibility(t *testing.T) { + f := newMessageAttachmentFixture(t) + key := uuid.NewString() + first := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, map[string]any{"text": "Legacy text 👋"})) + second := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody(first.Text, []string{}))) + if first.ChatID != f.chat.ID || first.SenderID != f.alice || first.Text != "Legacy text 👋" || !reflect.DeepEqual(first, second) { + t.Fatal("absent attachment_ids and [] must preserve the same text-only Message") + } + f.assertLinks(t, first, []string{}) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("text-only replay changed or duplicated history") + } +} + +func TestMessageAttachmentsOnlyAndOrderZeroToTen(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for count := 0; count <= 10; count++ { + t.Run(fmt.Sprintf("count_%d", count), func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids := append([]string{}, f.ids[:count]...) + text := "" + if count == 0 { + text = "Text without attachments" + } + key := uuid.NewString() + body := attachmentBody(text, ids) + if count == 1 { + body = attachmentBody(text, []string{strings.ToUpper(ids[0])}) + } + first := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)) + if count == 1 { + before := f.snapshot(t) + if replay := decodeAttachmentMessage(t, f.call(f.bob, "POST", f.chat.ID, key, body)); !reflect.DeepEqual(replay, first) { + t.Fatal("identical JSON with uppercase attachment UUID must replay the same 201/Message") + } + f.assertUnchanged(t, before) + } + if first.Text != text || first.SenderID != f.bob || first.ChatID != f.chat.ID { + t.Fatal("message-only/attachment-only content or sender changed") + } + f.assertLinks(t, first, ids) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("history lost the message or attachment order") + } + }) + } +} + +func TestMessageAttachmentsInvalidReferencesAreAtomic(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for _, kind := range []string{"unknown_id", "attachment_from_other_chat", "foreign_chat"} { + t.Run(kind, func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids, actor, chatID := []string{f.ids[0], uuid.NewString()}, f.alice, f.chat.ID + if kind == "attachment_from_other_chat" { + ids[1] = f.foreign[0] + } else if kind == "foreign_chat" { + actor, chatID, ids = f.bob, f.other.ID, []string{f.foreign[0]} + } + before, key := f.snapshot(t), uuid.NewString() + response := f.call(actor, "POST", chatID, key, attachmentBody("Must not be written", ids)) + assertContractError(t, response, 404, "not_found") + f.assertUnchanged(t, before) + // A rejected attempt must not reserve this user's key as a final result. + valid := decodeAttachmentMessage(t, f.call(actor, "POST", f.chat.ID, key, attachmentBody("Valid after 404", f.ids[:1]))) + f.assertLinks(t, valid, f.ids[:1]) + }) + } +} + +func TestMessageAttachmentsInvalidListLeavesNoRows(t *testing.T) { + if os.Getenv("LAB4_TEST_DATABASE_URL") == "" { + t.Skip("PENDING_INFRA: set LAB4_TEST_DATABASE_URL for dedicated PostgreSQL") + } + for _, kind := range []string{"duplicate", "eleven"} { + t.Run(kind, func(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids := f.ids + if kind == "duplicate" { + ids = []string{f.ids[0], f.ids[0]} + } + before := f.snapshot(t) + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, uuid.NewString(), attachmentBody("invalid list", ids)), 400, "invalid_request") + f.assertUnchanged(t, before) + }) + } +} + +func TestMessageAttachmentsIdempotencyIncludesListAndOrder(t *testing.T) { + f := newMessageAttachmentFixture(t) + key, ids := uuid.NewString(), f.ids[:2] + upperIDs := []string{strings.ToUpper(ids[0]), strings.ToUpper(ids[1])} + if slices.Equal(upperIDs, ids) { + t.Fatal("normalization fixture must contain a UUID with hexadecimal letters") + } + body := attachmentBody("Same text", upperIDs) + first := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, body)) + before := f.snapshot(t) + for range 3 { + if replay := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, body)); !reflect.DeepEqual(replay, first) { + t.Fatal("same key/body did not return the same Message") + } + f.assertUnchanged(t, before) + } + for _, changed := range [][]string{{ids[1], ids[0]}, {ids[0], f.ids[2]}, {ids[0]}, {}} { + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Same text", changed)), 409, "conflict") + f.assertUnchanged(t, before) + } + assertContractError(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Changed text", ids)), 409, "conflict") + f.assertUnchanged(t, before) + f.assertLinks(t, first, ids) + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("idempotency conflict changed history") + } +} + +func TestMessageAttachmentsConcurrentReplayDoesNotMultiplyLinks(t *testing.T) { + f := newMessageAttachmentFixture(t) + const workers = 8 + key, ids := uuid.NewString(), f.ids[:3] + gate := make(chan struct{}) + results := make(chan *httptest.ResponseRecorder, workers) + var group sync.WaitGroup + for range workers { + group.Add(1) + go func() { + defer group.Done() + <-gate + results <- f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Concurrent attachment replay", ids)) + }() + } + close(gate) + group.Wait() + close(results) + var first attachmentMessage + for response := range results { + message := decodeAttachmentMessage(t, response) + if first.ID == "" { + first = message + } else if !reflect.DeepEqual(message, first) { + t.Fatal("concurrent replay returned a different Message") + } + } + f.assertLinks(t, first, ids) + state := f.snapshot(t) + if len(state["messages"]) != 1 || len(state["messages_attachments"]) != len(ids) || len(state["message_idempotency"]) != 1 { + t.Fatal("concurrent replay multiplied message, link or idempotency rows") + } + if got := f.history(t); len(got) != 1 || !sameHistoryMessage(got[0], first) { + t.Fatal("concurrent replay changed history") + } +} + +func TestMessageAttachmentsLinkFailureRollsBackEverything(t *testing.T) { + f := newMessageAttachmentFixture(t) + ids, key := f.ids[:2], uuid.NewString() + // The sequence survives rollback. It proves that the fault was reached only + // after the first link was visible inside the actual application transaction. + _, err := f.pool.Exec(f.ctx, `CREATE SEQUENCE test_link_fault_reached; + CREATE FUNCTION fail_second_test_link() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF (SELECT count(*) FROM messages_attachments WHERE message_id=NEW.message_id) >= 1 THEN + PERFORM nextval('test_link_fault_reached'); + RAISE EXCEPTION 'synthetic link failure'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER test_link_fault BEFORE INSERT ON messages_attachments + FOR EACH ROW EXECUTE FUNCTION fail_second_test_link();`) + if err != nil { + t.Fatal("cannot prepare isolated link fault") + } + before := f.snapshot(t) + response := f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Rollback all writes", ids)) + assertContractError(t, response, 500, "internal_error") + var faultReached bool + if f.pool.QueryRow(f.ctx, "SELECT is_called FROM test_link_fault_reached").Scan(&faultReached) != nil || !faultReached { + t.Fatal("test did not reach an injected link failure after one real link insert") + } + f.assertUnchanged(t, before) + if strings.Contains(response.Body.String(), "synthetic link failure") { + t.Fatal("response exposed internal SQL fault details") + } + if _, err := f.pool.Exec(f.ctx, "DROP TRIGGER test_link_fault ON messages_attachments"); err != nil { + t.Fatal("cannot clear isolated link fault") + } + message := decodeAttachmentMessage(t, f.call(f.alice, "POST", f.chat.ID, key, attachmentBody("Rollback all writes", ids))) + f.assertLinks(t, message, ids) + state := f.snapshot(t) + if len(state["messages"]) != 1 || len(state["messages_attachments"]) != 2 || len(state["message_idempotency"]) != 1 { + t.Fatal("retry after transient link failure did not commit exactly one complete message") + } +} diff --git a/student-tests/auth_tests/password_test.go b/student-tests/auth_tests/password_test.go new file mode 100644 index 00000000..983cda77 --- /dev/null +++ b/student-tests/auth_tests/password_test.go @@ -0,0 +1,126 @@ +package auth_tests + +import ( + "errors" + "strings" + "testing" + + "highload/internal/security" +) + +// These are tests of the selected Argon2id implementation, not a requirement +// that lab2 use particular parameters or a particular encoded hash format. +func TestPasswordRoundTripAndRandomSalt(t *testing.T) { + password := "Синтетический:пароль:😀" + first, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + second, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + if first == password || second == password || first == second { + t.Fatal("hash must differ from plaintext and have a fresh random salt") + } + if !strings.HasPrefix(first, "$argon2id$") { + t.Fatal("selected password implementation did not encode Argon2id") + } + for _, encoded := range []string{first, second} { + ok, err := security.VerifyPassword(password, encoded) + if err != nil || !ok { + t.Fatal("stored hash does not verify the original password") + } + ok, err = security.VerifyPassword(password+"x", encoded) + if err != nil || ok { + t.Fatal("wrong password was accepted or verifier failed") + } + } +} + +func TestPasswordDoesNotTruncateLongUTF8(t *testing.T) { + // Both passwords share much more than bcrypt's 72-byte limit; each is + // within the lab's 128-code-point maximum. + password := strings.Repeat("😀", 127) + "A" + different := strings.Repeat("😀", 127) + "B" + encoded, err := security.HashPassword(password) + if err != nil { + t.Fatal("maximum-length UTF-8 password cannot be hashed") + } + ok, err := security.VerifyPassword(password, encoded) + if err != nil || !ok { + t.Fatal("maximum-length UTF-8 password cannot be verified") + } + ok, err = security.VerifyPassword(different, encoded) + if err != nil || ok { + t.Fatal("password suffix was truncated") + } +} + +func TestPasswordWhitespaceIsSignificant(t *testing.T) { + password := " synthetic-password " + encoded, err := security.HashPassword(password) + if err != nil { + t.Fatal("password hashing failed") + } + ok, err := security.VerifyPassword(strings.TrimSpace(password), encoded) + if err != nil || ok { + t.Fatal("password whitespace was discarded") + } +} + +func TestMalformedPasswordHashRejectedWithoutPanic(t *testing.T) { + valid, err := security.HashPassword("synthetic-password") + if err != nil { + t.Fatal("password hashing failed") + } + parts := strings.Split(valid, "$") + if len(parts) != 6 { + t.Fatal("unexpected encoded hash format") + } + cases := map[string]string{ + "empty": "", + "plaintext": "synthetic-password", + "wrong_algorithm": strings.Replace(valid, "argon2id", "argon2i", 1), + "wrong_version": strings.Replace(valid, "$v=19$", "$v=999$", 1), + "missing_version": strings.Replace(valid, "$v=19$", "$19$", 1), + "extra_field": valid + "$extra", + } + for name, params := range map[string]string{ + "zero_iterations": "m=65536,t=0,p=2", + "zero_threads": "m=65536,t=3,p=0", + "tiny_memory": "m=1,t=3,p=2", + "huge_memory": "m=4294967295,t=3,p=2", + "huge_iterations": "m=65536,t=4294967295,p=2", + "huge_threads": "m=65536,t=3,p=255", + "negative": "m=-1,t=3,p=2", + "non_numeric": "m=no,t=3,p=2", + "missing_param": "m=65536,t=3", + } { + copyParts := append([]string(nil), parts...) + copyParts[3] = params + cases[name] = strings.Join(copyParts, "$") + } + for name, field := range map[string]int{"salt": 4, "hash": 5} { + for label, value := range map[string]string{ + "empty": "", "bad_base64": "!!!", "wrong_size": "YQ", + } { + copyParts := append([]string(nil), parts...) + copyParts[field] = value + cases[name+"_"+label] = strings.Join(copyParts, "$") + } + } + for name, encoded := range cases { + t.Run(name, func(t *testing.T) { + defer func() { + if recover() != nil { + t.Error("malformed stored hash caused a panic") + } + }() + ok, err := security.VerifyPassword("synthetic-password", encoded) + if ok || !errors.Is(err, security.ErrInvalidHash) { + t.Error("malformed stored hash was not rejected") + } + }) + } +} diff --git a/student-tests/auth_tests/redis_test.go b/student-tests/auth_tests/redis_test.go new file mode 100644 index 00000000..b3fc2ec3 --- /dev/null +++ b/student-tests/auth_tests/redis_test.go @@ -0,0 +1,471 @@ +package auth_tests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "testing" + "time" + + "highload/internal/config" + "highload/internal/database" + repoerrors "highload/internal/repository" + "highload/internal/repository/postgres" + redisrepo "highload/internal/repository/redis" + "highload/internal/service" + httptransport "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/google/uuid" + redisclient "github.com/redis/go-redis/v9" +) + +func TestRedisAddressIsAnAddressNotAFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "postgres_url") + if err := os.WriteFile(path, []byte("postgres://fixture:fixture@postgres/messenger"), 0600); err != nil { + t.Fatal("cannot create synthetic config fixture") + } + t.Setenv("DATABASE_URL_FILE", path) + t.Setenv("REDIS_ADDR", "redis:6379") + t.Setenv("SESSION_TTL_SECONDS", "3600") + t.Setenv("SESSION_STORE", "redis") + t.Setenv("S3_ENDPOINT", "http://s3-fixture.invalid") + t.Setenv("S3_REGION", "us-east-1") + t.Setenv("S3_BUCKET", "health-fixture") + for variable, value := range map[string]string{ + "S3_ACCESS_KEY_ID_FILE": "fixture-access-key", + "S3_SECRET_ACCESS_KEY_FILE": "fixture-secret-key", + } { + keyFile := filepath.Join(dir, variable) + if err := os.WriteFile(keyFile, []byte(value), 0600); err != nil { + t.Fatal("cannot create synthetic S3 key file") + } + t.Setenv(variable, keyFile) + } + cfg, err := config.Load() + if err != nil { + t.Fatalf("cannot load synthetic config with REDIS_ADDR=redis:6379: %v", err) + } + if cfg.Database.RedisAddr != "redis:6379" { + t.Fatal("REDIS_ADDR did not reach the Redis configuration") + } +} + +type redisFixture struct { + ctx context.Context + client *redisclient.Client + repo *redisrepo.SessionRepository + keys []string + audit *sessionWriteAudit +} + +// Only observe writes through this test's client. No global key counting or +// FLUSH commands: other clients' sessions must not affect assertions/cleanup. +type sessionWriteAudit struct { + sync.Mutex + keys map[string]bool + writes int +} + +func (a *sessionWriteAudit) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (a *sessionWriteAudit) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (a *sessionWriteAudit) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + if cmd.Name() == "set" && len(cmd.Args()) > 1 { + if key, ok := cmd.Args()[1].(string); ok { + a.Lock() + a.keys[key] = true + a.writes++ + a.Unlock() + } + } + return next(ctx, cmd) + } +} +func (a *sessionWriteAudit) writeCount() int { a.Lock(); defer a.Unlock(); return a.writes } +func (a *sessionWriteAudit) ownedKeys() []string { + a.Lock() + defer a.Unlock() + keys := make([]string, 0, len(a.keys)) + for key := range a.keys { + keys = append(keys, key) + } + return keys +} + +type aofReplyHook struct { + reply any + err error +} + +func (h aofReplyHook) DialHook(next redisclient.DialHook) redisclient.DialHook { return next } +func (h aofReplyHook) ProcessPipelineHook(next redisclient.ProcessPipelineHook) redisclient.ProcessPipelineHook { + return next +} +func (h aofReplyHook) ProcessHook(next redisclient.ProcessHook) redisclient.ProcessHook { + return func(ctx context.Context, cmd redisclient.Cmder) error { + if cmd.Name() == "waitaof" { + // DEL still runs on the real test Redis. Only its persistence + // acknowledgement is fault-injected deterministically. + value := cmd.(*redisclient.Cmd) + value.SetVal(h.reply) + value.SetErr(h.err) + return h.err + } + return next(ctx, cmd) + } +} + +func TestRedisUnconfirmedAOFNeverReturnsSuccessfulLogout(t *testing.T) { + for _, tc := range []struct { + name string + reply any + err error + }{ + {"local_not_fsynced", []any{int64(0), int64(0)}, nil}, + {"only_replica_fsynced", []any{int64(0), int64(1)}, nil}, + {"malformed_reply", []any{int64(1)}, nil}, + {"command_failure", nil, errors.New("synthetic acknowledgement failure")}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + f.redis.client.AddHook(aofReplyHook{tc.reply, tc.err}) + request := httptest.NewRequest(http.MethodDelete, "/api/v1/auth/sessions/current", nil) + request.Header.Set("Authorization", "Bearer "+session.Token) + recorder := httptest.NewRecorder() + redisRouter(f, f.redis.client).ServeHTTP(recorder, request) + if recorder.Code != 500 && recorder.Code != 503 { + t.Fatalf("unconfirmed persistence must fail logout, got HTTP %d", recorder.Code) + } + if (tc.name == "local_not_fsynced" || tc.name == "only_replica_fsynced") && recorder.Code != 503 { + t.Errorf("valid reply without local acknowledgement must be HTTP 503, got %d", recorder.Code) + } + code := "internal_error" + if recorder.Code == 503 { + code = "unavailable" + } + assertContractError(t, recorder, recorder.Code, code) + hash := sha256.Sum256([]byte(session.Token)) + if f.redis.client.Exists(f.ctx, "session:"+hex.EncodeToString(hash[:])).Val() != 0 { + t.Fatal("fault fixture did not actually execute DEL before WAITAOF") + } + }) + } +} + +func newRedisFixture(t *testing.T) *redisFixture { + t.Helper() + addr := os.Getenv("AUTH_TEST_REDIS_ADDR") + if addr == "" { + t.Skip("Redis not verified: set AUTH_TEST_REDIS_ADDR to a dedicated test Redis with AOF") + } + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + t.Cleanup(cancel) + client, err := database.NewRedis(ctx, addr) + if err != nil { + t.Fatal("dedicated test Redis is unavailable") + } + audit := &sessionWriteAudit{keys: make(map[string]bool)} + client.AddHook(audit) + f := &redisFixture{ctx: ctx, client: client, repo: redisrepo.NewSessionRepository(client), audit: audit} + t.Cleanup(func() { + cleanup, done := context.WithTimeout(context.Background(), 3*time.Second) + defer done() + // No FLUSHDB/FLUSHALL: delete only this test's own random token keys. + keys := append(f.audit.ownedKeys(), f.keys...) + if len(keys) > 0 { + if err := client.Del(cleanup, keys...).Err(); err != nil { + t.Error("test-owned Redis key cleanup failed") + } + } + _ = client.Close() + }) + return f +} + +func (f *redisFixture) track(token string) [32]byte { + hash := sha256.Sum256([]byte(token)) + f.keys = append(f.keys, "session:"+hex.EncodeToString(hash[:])) + return hash +} + +func TestRedisSessionHasTTLAndReadsDoNotExtendIt(t *testing.T) { + f := newRedisFixture(t) + hash := f.track("synthetic-" + uuid.NewString()) + created, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 5*time.Second) + if err != nil { + t.Fatal("cannot create Redis session") + } + key := f.keys[0] + before := f.client.PTTL(f.ctx, key).Val() + if before <= 0 || before > 5*time.Second { + t.Fatal("session key has no bounded positive TTL") + } + time.Sleep(120 * time.Millisecond) + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || loaded == nil { + t.Fatal("cannot read Redis session") + } + if loaded.UserID != created.UserID || !loaded.ExpiresAt.Equal(created.ExpiresAt) || + !loaded.CreatedAt.Equal(created.CreatedAt) { + t.Fatal("session identity or original timestamps changed on read") + } + after := f.client.PTTL(f.ctx, key).Val() + if after <= 0 || after >= before { + t.Fatal("reading refreshed or removed the session TTL") + } +} + +func TestRedisExpiredAndUnknownSessionsAreNotFound(t *testing.T) { + f := newRedisFixture(t) + hash := f.track(uuid.NewString()) + if _, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("unknown session must be not found") + } + if _, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 200*time.Millisecond); err != nil { + t.Fatal("cannot create short-lived test session") + } + time.Sleep(350 * time.Millisecond) + if _, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("expired Redis session is still available") + } +} + +func TestRedisLogoutOnlyRemovesSelectedSession(t *testing.T) { + f := newRedisFixture(t) + first, second := f.track(uuid.NewString()), f.track(uuid.NewString()) + userID := uuid.NewString() + for _, hash := range [][32]byte{first, second} { + if _, err := f.repo.Create(f.ctx, hash[:], userID, time.Minute); err != nil { + t.Fatal("cannot create independent sessions") + } + } + count, err := f.repo.RevokeSession(f.ctx, first[:]) + if err != nil || count != 1 { + t.Fatal("logout failed to remove exactly one session or acknowledge persistence") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, first[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("revoked session is still readable") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, second[:]); err != nil { + t.Fatal("logout removed another session") + } +} + +func TestRedisSessionsSurviveActualRestartWithoutExtendingTTL(t *testing.T) { + hook := os.Getenv("AUTH_TEST_REDIS_RESTART_HOOK") + if hook == "" { + t.Skip("Redis restart not verified: set an explicit dedicated test restart hook") + } + f := newRedisFixture(t) + active, revoked := f.track(uuid.NewString()), f.track(uuid.NewString()) + created, err := f.repo.Create(f.ctx, active[:], uuid.NewString(), 30*time.Second) + if err != nil { + t.Fatal("cannot create active session") + } + if _, err := f.repo.Create(f.ctx, revoked[:], created.UserID, time.Minute); err != nil { + t.Fatal("cannot create revocation fixture") + } + if _, err := f.repo.RevokeSession(f.ctx, revoked[:]); err != nil { + t.Fatal("cannot persist logout before restart") + } + before := f.client.PTTL(f.ctx, f.keys[0]).Val() + if err := exec.CommandContext(f.ctx, hook).Run(); err != nil { + t.Fatal("explicit Redis restart hook failed (private output suppressed)") + } + deadline := time.Now().Add(5 * time.Second) + for f.client.Ping(f.ctx).Err() != nil && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, active[:]) + if err != nil || loaded == nil || !loaded.ExpiresAt.Equal(created.ExpiresAt) { + t.Fatal("restart lost active session or changed its original deadline") + } + if after := f.client.PTTL(f.ctx, f.keys[0]).Val(); after <= 0 || after >= before { + t.Fatal("restart extended or lost remaining TTL") + } + if _, err := f.repo.GetSessionByTokenHash(f.ctx, revoked[:]); !errors.Is(err, repoerrors.ErrNotFound) { + t.Fatal("restart revived a revoked session") + } +} + +func TestRedisActiveSessionSurvivesRestartIndependentlyOfLogout(t *testing.T) { + hook := os.Getenv("AUTH_TEST_REDIS_RESTART_HOOK") + if hook == "" { + t.Skip("Redis restart not verified: set an explicit dedicated test restart hook") + } + f := newRedisFixture(t) + hash := f.track(uuid.NewString()) + created, err := f.repo.Create(f.ctx, hash[:], uuid.NewString(), 30*time.Second) + if err != nil { + t.Fatal("cannot create active session") + } + before := f.client.PTTL(f.ctx, f.keys[0]).Val() + if err := exec.CommandContext(f.ctx, hook).Run(); err != nil { + t.Fatal("explicit Redis restart hook failed (private output suppressed)") + } + deadline := time.Now().Add(5 * time.Second) + for f.client.Ping(f.ctx).Err() != nil && time.Now().Before(deadline) { + time.Sleep(50 * time.Millisecond) + } + loaded, err := f.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || loaded == nil || !loaded.ExpiresAt.Equal(created.ExpiresAt) || loaded.UserID != created.UserID { + t.Fatal("restart lost active session identity or changed its deadline") + } + if after := f.client.PTTL(f.ctx, f.keys[0]).Val(); after <= 0 || after >= before { + t.Fatal("restart extended or removed the session TTL") + } +} + +func refusedRedis(t *testing.T) *redisclient.Client { + t.Helper() + client := redisclient.NewClient(&redisclient.Options{ + Addr: "synthetic-unavailable:6379", MaxRetries: -1, + Dialer: func(context.Context, string, string) (net.Conn, error) { + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} + }, + }) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +func TestRedisUnavailableKeepsUnavailableCategory(t *testing.T) { + repo := redisrepo.NewSessionRepository(refusedRedis(t)) + hash := sha256.Sum256([]byte("synthetic-failed-token")) + t.Run("create", func(t *testing.T) { + _, err := repo.Create(context.Background(), hash[:], uuid.NewString(), time.Minute) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + t.Run("read", func(t *testing.T) { + _, err := repo.GetSessionByTokenHash(context.Background(), hash[:]) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + t.Run("logout", func(t *testing.T) { + _, err := repo.RevokeSession(context.Background(), hash[:]) + if !errors.Is(err, repoerrors.ErrServiceUnavailable) { + t.Fatalf("want service unavailable, got %v", err) + } + if err != nil && !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("Redis network diagnostic context was lost") + } + }) + auth := service.NewAuthService(nil, repo, time.Hour) + _, err := auth.ValidateSession(context.Background(), "synthetic-error-context") + if !errors.Is(err, service.ErrServiceUnavailable) || !strings.Contains(err.Error(), syscall.ECONNREFUSED.Error()) { + t.Error("service lost Redis unavailable category or diagnostic context") + } + +} + +func redisRouter(f *fixture, client *redisclient.Client) http.Handler { + users := postgres.NewUserRepository(f.queries) + auth := service.NewAuthService(users, redisrepo.NewSessionRepository(client), time.Hour) + userService := service.NewUserService(users) + chats := service.NewChatService(postgres.NewChatRepository(f.queries, f.pool), users, userService) + messages := service.NewMessageService(postgres.NewMessageRepository(f.queries, f.pool), chats, userService) + objects := readyS3Storage() + attachments := service.NewAttachmentService(postgres.NewAttachmentRepository(f.queries, f.pool), chats, objects) + return httptransport.NewRouter(middleware.NewMiddleware(auth, "auth-test"), handler.NewAuthHandler(auth), + handler.NewHealthHandler(f.pool, client, objects), handler.NewChatHandler(chats), handler.NewUserHandler(userService), + handler.NewMessageHandler(messages), handler.NewAttachmentHandler(attachments)).Handler() +} + +func TestRedisCriticalOutageHTTPAndReadiness(t *testing.T) { + f := newFixture(t) + router := redisRouter(f, refusedRedis(t)) + for _, tc := range []struct { + name, path string + status int + }{ + {"liveness", "/health/live", 200}, + {"readiness", "/health/ready", 503}, + {"protected_request", "/api/v1/users/me", 503}, + } { + t.Run(tc.name, func(t *testing.T) { + request := httptest.NewRequest(http.MethodGet, tc.path, nil) + request.Header.Set("Authorization", "Bearer synthetic-outage-token") + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + if recorder.Code != tc.status { + t.Fatalf("want HTTP %d, got %d", tc.status, recorder.Code) + } + if tc.status == 503 { + assertContractError(t, recorder, 503, "unavailable") + } + }) + } +} + +func TestRedisPublicHTTPContractIsUnchanged(t *testing.T) { + f := newFixture(t) + r := newRedisFixture(t) + server := httptest.NewServer(redisRouter(f, r.client)) + t.Cleanup(server.Close) + _, source, _, _ := runtime.Caller(0) + root := filepath.Clean(filepath.Join(filepath.Dir(source), "..", "..")) + // The full acceptance suite performs many Argon2id operations. Under -race + // it needs its own budget rather than the 45-second SQL fixture budget. + for _, tc := range []struct { + name string + args []string + }{ + {"teacher_smoke", []string{filepath.Join(root, "tests/smoke.py"), "--base-url", server.URL}}, + {"auth_normal", []string{filepath.Join(root, "student-tests/auth_contract.py"), "--base-url", server.URL, "--ttl", "3600"}}, + } { + t.Run(tc.name, func(t *testing.T) { + // Each independent subprocess gets the full acceptance budget. A smoke + // run must not consume part of the following auth suite's allowance. + acceptanceCtx, cancel := context.WithTimeout(context.Background(), 300*time.Second) + defer cancel() + cmd := exec.CommandContext(acceptanceCtx, "python3", tc.args...) + cmd.Dir = root + output, err := cmd.CombinedOutput() + if err != nil { + for _, line := range strings.Split(string(output), "\n") { + if strings.HasPrefix(line, "test_") { + // Only method names: never print credentials or subtest values. + name, _, _ := strings.Cut(line, " ") + t.Log("acceptance progress: " + name) + } + } + if acceptanceCtx.Err() != nil { + t.Fatal("HTTP acceptance subprocess exceeded its own execution budget; application operation budgets are unchanged") + } + // Public clients keep tokens in memory. Show only status diagnostics. + for _, line := range strings.Split(string(output), "\n") { + if strings.Contains(line, "expected [") && strings.Contains(line, "received") { + t.Log(line) + } + } + t.Fatal("HTTP acceptance contract failed with Redis-backed sessions") + } + }) + } +} diff --git a/student-tests/auth_tests/restart_http_test.go b/student-tests/auth_tests/restart_http_test.go new file mode 100644 index 00000000..c85b1825 --- /dev/null +++ b/student-tests/auth_tests/restart_http_test.go @@ -0,0 +1,121 @@ +package auth_tests + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "net/http" + "os" + "os/exec" + "strings" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestRedisHTTPFullRestartKeepsDeadlineAndRevocation(t *testing.T) { + base, hook := os.Getenv("AUTH_TEST_BASE_URL"), os.Getenv("AUTH_TEST_STACK_RESTART_HOOK") + if base == "" || hook == "" { + t.Skip("set AUTH_TEST_BASE_URL and AUTH_TEST_STACK_RESTART_HOOK for a disposable API+stores stack") + } + for _, withLogout := range []bool{false, true} { + name := "active_only" + if withLogout { + name = "active_and_revoked" + } + t.Run(name, func(t *testing.T) { + f := newRedisFixture(t) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + client := &http.Client{Timeout: 5 * time.Second} + username := "restart_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:18] + password := "Synthetic:" + uuid.NewString() + call := func(method, path, token string, body any, basic bool, want int, dst any) { + t.Helper() + var data bytes.Buffer + if body != nil { + if json.NewEncoder(&data).Encode(body) != nil { + t.Fatal("cannot encode test request") + } + } + req, err := http.NewRequestWithContext(ctx, method, strings.TrimRight(base, "/")+path, &data) + if err != nil { + t.Fatal("cannot construct test request") + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if basic { + req.SetBasicAuth(username, password) + } else if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + response, err := client.Do(req) + if err != nil { + t.Fatal("restart HTTP request failed") + } + defer response.Body.Close() + if response.StatusCode != want { + t.Fatalf("%s %s: want %d, got %d", method, path, want, response.StatusCode) + } + if dst != nil && json.NewDecoder(response.Body).Decode(dst) != nil { + t.Fatal("invalid restart response JSON") + } + } + call("POST", "/api/v1/users", "", map[string]string{"username": username, "password": password, "display_name": "Restart fixture"}, false, 201, nil) + var active struct { + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` + } + call("POST", "/api/v1/auth/sessions", "", nil, true, 201, &active) + hash := f.track(active.Token) + before, err := f.repo.GetSessionByTokenHash(ctx, hash[:]) + if err != nil { + t.Fatal("cannot inspect issued session") + } + if !before.ExpiresAt.Equal(active.ExpiresAt) { + t.Fatal("HTTP and Redis expiry differ") + } + remaining := f.client.PTTL(ctx, f.keys[0]).Val() + if remaining < 2*time.Minute { + t.Fatal("normal restart suite requires long session TTL") + } + revoked := "" + if withLogout { + var other struct { + Token string `json:"token"` + } + call("POST", "/api/v1/auth/sessions", "", nil, true, 201, &other) + revoked = other.Token + f.track(revoked) + call("DELETE", "/api/v1/auth/sessions/current", revoked, nil, false, 204, nil) + call("GET", "/api/v1/users/me", revoked, nil, false, 401, nil) + } + if exec.CommandContext(ctx, hook).Run() != nil { + t.Fatal("real API+stores restart hook failed") + } + call("GET", "/health/live", "", nil, false, 200, nil) + call("GET", "/health/ready", "", nil, false, 200, nil) + call("GET", "/api/v1/users/me", active.Token, nil, false, 200, nil) + after, err := f.repo.GetSessionByTokenHash(ctx, hash[:]) + if err != nil { + t.Fatal("restart lost session") + } + if !after.ExpiresAt.Equal(before.ExpiresAt) || after.UserID != before.UserID || !after.CreatedAt.Equal(before.CreatedAt) { + t.Fatal("actual API+Redis restart changed session identity/deadline") + } + if ttl := f.client.PTTL(ctx, f.keys[0]).Val(); ttl <= 0 || ttl >= remaining { + t.Fatal("restart reset/removed original TTL") + } + if revoked != "" { + call("GET", "/api/v1/users/me", revoked, nil, false, 401, nil) + revokedHash := sha256.Sum256([]byte(revoked)) + if _, err = f.repo.GetSessionByTokenHash(ctx, revokedHash[:]); err == nil { + t.Fatal("revoked credential reappeared after restart") + } + } + }) + } +} diff --git a/student-tests/auth_tests/storage_test.go b/student-tests/auth_tests/storage_test.go new file mode 100644 index 00000000..87dca4a2 --- /dev/null +++ b/student-tests/auth_tests/storage_test.go @@ -0,0 +1,345 @@ +package auth_tests + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "testing" + "time" + + "highload/internal/database/sqlc" + "highload/internal/model" + repository "highload/internal/repository/postgres" + redisrepo "highload/internal/repository/redis" + "highload/internal/security" + "highload/internal/service" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type fixture struct { + admin *pgxpool.Pool + pool *pgxpool.Pool + config *pgxpool.Config + queries *sqlc.Queries + users *service.UserService + auth *service.AuthService + redis *redisFixture + ctx context.Context +} + +func newFixture(t *testing.T) *fixture { + t.Helper() + f := newDatabaseFixture(t, "AUTH_TEST_DATABASE_URL", 4) + f.redis = newRedisFixture(t) + userRepo := repository.NewUserRepository(f.queries) + f.users = service.NewUserService(userRepo) + f.auth = service.NewAuthService(userRepo, redisrepo.NewSessionRepository(f.redis.client), time.Hour) + return f +} + +// Shared PostgreSQL fixture: attachment transaction checks do not require Redis/S3. +func newDatabaseFixture(t *testing.T, environment string, maxConnections int32) *fixture { + t.Helper() + url := os.Getenv(environment) + if url == "" { + t.Skip("PENDING_INFRA: set " + environment + " for a dedicated test PostgreSQL") + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + t.Cleanup(cancel) + config, err := pgxpool.ParseConfig(url) + if err != nil { + t.Fatal("cannot parse test PostgreSQL configuration (credentials suppressed)") + } + config.MaxConns = maxConnections + config.MinConns = 0 + config.ConnConfig.ConnectTimeout = 3 * time.Second + config.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + admin, err := pgxpool.NewWithConfig(ctx, config) + if err != nil { + t.Fatal("cannot open test PostgreSQL pool") + } + t.Cleanup(admin.Close) + if err := admin.Ping(ctx); err != nil { + t.Fatal("test PostgreSQL is unavailable") + } + // Only this uniquely named schema is created and dropped. Existing tables + // and volumes are never removed. The role needs CREATE SCHEMA permission. + schema := "auth_test_" + strings.ReplaceAll(uuid.NewString(), "-", "") + quoted := pgx.Identifier{schema}.Sanitize() + if _, err := admin.Exec(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatal("cannot create isolated test schema") + } + t.Cleanup(func() { + cleanupCtx, done := context.WithTimeout(context.Background(), 5*time.Second) + defer done() + if _, err := admin.Exec(cleanupCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Error("isolated test schema cleanup failed") + } + }) + config = config.Copy() + config.ConnConfig.RuntimeParams["search_path"] = schema + pool, err := pgxpool.NewWithConfig(ctx, config.Copy()) + if err != nil { + t.Fatal("cannot connect to isolated test schema") + } + f := &fixture{admin: admin, config: config, ctx: ctx, pool: pool, queries: sqlc.New(pool)} + t.Cleanup(func() { f.pool.Close() }) + _, source, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("cannot locate migration files") + } + migrations, err := filepath.Glob(filepath.Join(filepath.Dir(source), "..", "..", "db", "migrations", "*.sql")) + if err != nil || len(migrations) == 0 { + t.Fatal("migration SQL is missing") + } + sort.Strings(migrations) + for _, path := range migrations { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal("cannot read migration SQL") + } + // Current migrations are ordinary SQL with Goose Up/Down markers. + // This applies schema fixtures; it is NOT a test of Goose version tracking. + up, _, _ := strings.Cut(string(data), "-- +goose Down") + if !strings.Contains(up, "-- +goose Up") { + t.Fatal("test fixture expects SQL migrations with Goose Up/Down markers") + } + if _, err := f.pool.Exec(ctx, up, pgx.QueryExecModeSimpleProtocol); err != nil { + t.Fatal("migration cannot initialize isolated schema") + } + } + return f +} + +func (f *fixture) connect(t *testing.T, ttl time.Duration) { + t.Helper() + pool, err := pgxpool.NewWithConfig(f.ctx, f.config.Copy()) + if err != nil { + t.Fatal("cannot connect to isolated schema") + } + f.pool = pool + f.queries = sqlc.New(pool) + userRepo := repository.NewUserRepository(f.queries) + sessionRepo := redisrepo.NewSessionRepository(f.redis.client) + f.users = service.NewUserService(userRepo) + f.auth = service.NewAuthService(userRepo, sessionRepo, ttl) +} + +func (f *fixture) user(t *testing.T) (*model.User, string) { + t.Helper() + password := "Synthetic:" + uuid.NewString() + username := "auth_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:18] + user, err := f.users.CreateUser(f.ctx, username, "Auth storage", password) + if err != nil { + t.Fatal("cannot create synthetic user") + } + return user, password +} + +func (f *fixture) session(t *testing.T, user *model.User, password string) *service.CreatedSession { + t.Helper() + session, err := f.auth.CreateSession(f.ctx, user.Username, password) + if err != nil { + t.Fatal("cannot create synthetic session") + } + return session +} + +func TestStoragePasswordIsKDFAndDuplicateCannotOverwrite(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + stored, err := f.queries.GetUserByUsername(f.ctx, user.Username) + if err != nil || stored.PasswordHash == password || !strings.HasPrefix(stored.PasswordHash, "$argon2id$") { + t.Fatal("database did not preserve the selected password KDF") + } + ok, err := security.VerifyPassword(password, stored.PasswordHash) + if err != nil || !ok { + t.Fatal("stored password does not verify") + } + _, err = f.users.CreateUser(f.ctx, user.Username, "Overwrite", password+"x") + if !errors.Is(err, service.ErrAlreadyExists) { + t.Fatal("duplicate username was not rejected") + } + after, err := f.queries.GetUserByUsername(f.ctx, user.Username) + if err != nil || after.ID != stored.ID || after.PasswordHash != stored.PasswordHash || + after.DisplayName != stored.DisplayName { + t.Fatal("failed duplicate registration changed the original user") + } +} + +func TestStorageSessionHashAndReconnectPreserveDeadline(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + hash := sha256.Sum256([]byte(session.Token)) + key := "session:" + hex.EncodeToString(hash[:]) + row, err := f.redis.repo.GetSessionByTokenHash(f.ctx, hash[:]) + value, getErr := f.redis.client.Get(f.ctx, key).Result() + if err != nil || getErr != nil || strings.Contains(key, session.Token) || strings.Contains(value, session.Token) { + t.Fatal("selected session storage did not use SHA-256 of the opaque token") + } + for _, ownedKey := range f.redis.audit.ownedKeys() { + storedValue, err := f.redis.client.Get(f.ctx, ownedKey).Result() + if err != nil || strings.Contains(ownedKey, session.Token) || strings.Contains(storedValue, session.Token) { + t.Fatal("raw session token was persisted in Redis") + } + } + if f.redis.client.Exists(f.ctx, "session:"+session.Token, session.Token).Val() != 0 { + t.Fatal("raw token was stored as a Redis key") + } + if ttl := f.redis.client.PTTL(f.ctx, key).Val(); ttl <= 0 || ttl > time.Hour { + t.Fatal("session must have a positive bounded Redis TTL") + } + if session.TokenType != "Bearer" || len(session.Token) < 32 || row.RevokedAt != nil || + !row.ExpiresAt.Equal(session.ExpiresAt) || !row.ExpiresAt.After(row.CreatedAt) { + t.Fatal("invalid stored session metadata") + } + beforeDeadline := row.ExpiresAt + for range 3 { + id, err := f.auth.ValidateSession(f.ctx, session.Token) + if err != nil || id != user.ID { + t.Fatal("valid session rejected or mapped to another user") + } + } + f.pool.Close() + // A different process configuration must not reset already stored expiry. + f.connect(t, 24*time.Hour) + id, err := f.auth.ValidateSession(f.ctx, session.Token) + if err != nil || id != user.ID { + t.Fatal("new pool/services lost the existing session") + } + after, err := f.redis.repo.GetSessionByTokenHash(f.ctx, hash[:]) + if err != nil || !after.ExpiresAt.Equal(beforeDeadline) { + t.Fatal("reads or reconnection extended session TTL") + } +} + +func TestStorageWrongCredentialsNeverCreateSession(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + writes := f.redis.audit.writeCount() + for _, credentials := range [][2]string{ + {user.Username, password + "x"}, {"unknown_auth_user", password}, + } { + _, err := f.auth.CreateSession(f.ctx, credentials[0], credentials[1]) + if !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("wrong credentials did not yield the same unauthorized error") + } + } + if f.redis.audit.writeCount() != writes { + t.Fatal("failed login created a session") + } + for _, token := range []string{"", uuid.NewString()} { + if _, err := f.auth.ValidateSession(f.ctx, token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("empty or unknown token accepted") + } + } +} + +func TestStorageRevocationIsPersistentAndSessionSpecific(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + first, second := f.session(t, user, password), f.session(t, user, password) + if first.Token == second.Token { + t.Fatal("two logins reused the same token") + } + if err := f.auth.RevokeSession(f.ctx, first.Token); err != nil { + t.Fatal("logout failed") + } + f.pool.Close() + f.connect(t, time.Hour) + if _, err := f.auth.ValidateSession(f.ctx, first.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("revoked session revived after reconnect") + } + id, err := f.auth.ValidateSession(f.ctx, second.Token) + if err != nil || id != user.ID { + t.Fatal("logout affected another session") + } +} + +func TestStorageExpiredSessionCannotRevive(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + session := f.session(t, user, password) + hash := sha256.Sum256([]byte(session.Token)) + key := "session:" + hex.EncodeToString(hash[:]) + if err := f.redis.client.PExpire(f.ctx, key, time.Millisecond).Err(); err != nil { + t.Fatal("cannot set an expired fixture") + } + time.Sleep(20 * time.Millisecond) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("expired session accepted") + } + f.pool.Close() + f.connect(t, 24*time.Hour) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("new TTL configuration revived an expired session") + } +} + +func TestStorageShortTTLExpiresAndReLoginWorks(t *testing.T) { + f := newFixture(t) + f.auth = service.NewAuthService(repository.NewUserRepository(f.queries), + redisrepo.NewSessionRepository(f.redis.client), 2*time.Second) + user, password := f.user(t) + session := f.session(t, user, password) + if remaining := time.Until(session.ExpiresAt); remaining <= 0 || remaining > 3*time.Second { + t.Fatal("two-second TTL was not used for the stored session") + } + if id, err := f.auth.ValidateSession(f.ctx, session.Token); err != nil || id != user.ID { + t.Fatal("fresh short-lived session was rejected") + } + time.Sleep(max(0, time.Until(session.ExpiresAt)/2)) + if _, err := f.auth.ValidateSession(f.ctx, session.Token); err != nil { + t.Fatal("session expired before its original deadline") + } + timer := time.NewTimer(max(0, time.Until(session.ExpiresAt)) + 100*time.Millisecond) + defer timer.Stop() + select { + case <-timer.C: + case <-f.ctx.Done(): + t.Fatal("expiry fixture exceeded its deadline") + } + if _, err := f.auth.ValidateSession(f.ctx, session.Token); !errors.Is(err, service.ErrUnauthorized) { + t.Fatal("short-lived session was accepted after its stored deadline") + } + fresh := f.session(t, user, password) + if fresh.Token == session.Token { + t.Fatal("re-login reused the expired token") + } + if id, err := f.auth.ValidateSession(f.ctx, fresh.Token); err != nil || id != user.ID { + t.Fatal("re-login did not produce a usable session") + } +} + +func TestStorageChatRollbackAfterMemberInsertFailure(t *testing.T) { + f := newFixture(t) + user, _ := f.user(t) + repo := repository.NewChatRepository(f.queries, f.pool) + // A valid first member is inserted before the nonexistent member fails. + // Calling the repository bypasses prevalidation and tests real rollback. + _, err := repo.CreateChat(f.ctx, "Must roll back", []string{user.ID, uuid.NewString()}) + if err == nil { + t.Fatal("chat with nonexistent member unexpectedly committed") + } + for _, table := range []string{"chats", "chat_members"} { + var count int + err := f.pool.QueryRow(f.ctx, "SELECT count(*) FROM "+table).Scan(&count) + if err != nil || count != 0 { + t.Fatal("failed transaction left partial chat or membership rows") + } + } + created, err := repo.CreateChat(f.ctx, "Valid after rollback", []string{user.ID}) + if err != nil || created == nil { + t.Fatal("rollback poisoned subsequent valid transaction") + } +} diff --git a/student-tests/auth_tests/transport_test.go b/student-tests/auth_tests/transport_test.go new file mode 100644 index 00000000..61132e95 --- /dev/null +++ b/student-tests/auth_tests/transport_test.go @@ -0,0 +1,226 @@ +package auth_tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "sync" + "syscall" + "testing" + "time" + + "highload/internal/database/sqlc" + repository "highload/internal/repository/postgres" + "highload/internal/service" + "highload/internal/storage/s3" + httptransport "highload/internal/transport/http" + "highload/internal/transport/http/handler" + "highload/internal/transport/http/middleware" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" +) + +const healthFixtureBucket = "health-fixture" + +type readyS3Client struct { + mu sync.Mutex + objects map[string][]byte +} + +func (s *readyS3Client) Do(req *http.Request) (*http.Response, error) { + bucketPath := "/" + healthFixtureBucket + response := &http.Response{StatusCode: http.StatusOK, Header: make(http.Header), Body: http.NoBody} + if req.Method == http.MethodHead && req.URL.Path == bucketPath { + return response, nil + } + if !strings.HasPrefix(req.URL.Path, bucketPath+"/attachments/") { + return nil, fmt.Errorf("unexpected path in S3 fixture") + } + s.mu.Lock() + defer s.mu.Unlock() + switch req.Method { + case http.MethodPut: + defer req.Body.Close() + data, err := io.ReadAll(req.Body) + if err != nil { + return nil, err + } + s.objects[req.URL.Path] = data + case http.MethodGet: + data, ok := s.objects[req.URL.Path] + if !ok { + response.StatusCode = http.StatusNotFound + return response, nil + } + response.Body = io.NopCloser(bytes.NewReader(data)) + response.ContentLength = int64(len(data)) + default: + return nil, fmt.Errorf("unexpected method in S3 fixture") + } + return response, nil +} + +// Supply local S3 responses for readiness and the attachment upload in the HTTP smoke test. +func readyS3Storage() *s3.Storage { + return &s3.Storage{ + Bucket: healthFixtureBucket, + Client: awss3.New(awss3.Options{ + Region: "us-east-1", + BaseEndpoint: aws.String("http://s3-fixture.invalid"), + UsePathStyle: true, + Credentials: aws.AnonymousCredentials{}, + HTTPClient: &readyS3Client{objects: make(map[string][]byte)}, + }), + } +} + +type unavailableDB struct{} +type unavailableRow struct{} + +func (unavailableRow) Scan(...any) error { + return &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNREFUSED} +} + +func (unavailableDB) Exec(context.Context, string, ...any) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, unavailableRow{}.Scan() +} + +func (unavailableDB) Query(context.Context, string, ...any) (pgx.Rows, error) { + return nil, unavailableRow{}.Scan() +} + +func (unavailableDB) QueryRow(context.Context, string, ...any) pgx.Row { + return unavailableRow{} +} + +func TestSQLUnavailableLoginStopsBeforeSessionCreation(t *testing.T) { + queries := sqlc.New(unavailableDB{}) + auth := service.NewAuthService(repository.NewUserRepository(queries), nil, time.Hour) + mid := middleware.NewMiddleware(auth, "auth-test") + request := httptest.NewRequest(http.MethodPost, "/api/v1/auth/sessions", nil) + request.SetBasicAuth("synthetic_user", "Synthetic:password") + recorder := httptest.NewRecorder() + mid.RequestID(http.HandlerFunc(handler.NewAuthHandler(auth).CreateSession)).ServeHTTP(recorder, request) + assertContractError(t, recorder, 503, "unavailable") +} + +func TestMiddlewareMalformedAuthWritesExactlyOneError(t *testing.T) { + // Invalid syntax must be rejected before asking an auth service or DB. + // A nil service deliberately makes an accidental dependency call visible. + mid := middleware.NewMiddleware(nil, "auth-test") + for _, header := range []string{"", "Basic invalid", "Bearer", "Bearer ", + "Bearer invalid token", "Digest invalid"} { + t.Run("malformed", func(t *testing.T) { + nextCalled := false + next := http.HandlerFunc(func(http.ResponseWriter, *http.Request) { nextCalled = true }) + request := httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil) + request.Header.Set("Authorization", header) + recorder := httptest.NewRecorder() + mid.RequestID(mid.Auth(next)).ServeHTTP(recorder, request) + if nextCalled || recorder.Code != http.StatusUnauthorized { + t.Fatal("malformed auth reached business handler or did not yield 401") + } + if recorder.Header().Get("WWW-Authenticate") != "Bearer realm=\"messenger\"" { + t.Fatal("missing contract Bearer challenge") + } + var payload struct { + Error struct { + Code string `json:"code"` + RequestID string `json:"request_id"` + } `json:"error"` + } + decoder := json.NewDecoder(recorder.Body) + if err := decoder.Decode(&payload); err != nil { + t.Fatal("401 is not one valid JSON object") + } + if err := decoder.Decode(new(any)); err != io.EOF { + t.Fatal("middleware wrote more than one response after 401") + } + if payload.Error.Code != "unauthorized" || payload.Error.RequestID == "" || + payload.Error.RequestID != recorder.Header().Get("X-Request-Id") { + t.Fatal("error request ID or code violates contract") + } + }) + } +} + +func TestStorageHTTPAuthRoutingAndLogout(t *testing.T) { + f := newFixture(t) + user, password := f.user(t) + userRepo := repository.NewUserRepository(f.queries) + chatRepo := repository.NewChatRepository(f.queries, f.pool) + chatService := service.NewChatService(chatRepo, userRepo, f.users) + messageService := service.NewMessageService(repository.NewMessageRepository(f.queries, f.pool), chatService, f.users) + objects := readyS3Storage() + attachmentService := service.NewAttachmentService(repository.NewAttachmentRepository(f.queries, f.pool), chatService, objects) + router := httptransport.NewRouter( + middleware.NewMiddleware(f.auth, "auth-test"), handler.NewAuthHandler(f.auth), + handler.NewHealthHandler(f.pool, f.redis.client, objects), + handler.NewChatHandler(chatService), handler.NewUserHandler(f.users), + handler.NewMessageHandler(messageService), + handler.NewAttachmentHandler(attachmentService), + ).Handler() + + call := func(method, path, token string, login bool) *httptest.ResponseRecorder { + t.Helper() + request := httptest.NewRequest(method, path, nil) + if login { + request.SetBasicAuth(user.Username, password) + } else if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + if recorder.Header().Get("X-Request-Id") == "" { + t.Fatal("auth response missing X-Request-Id") + } + return recorder + } + defer func() { + if recover() != nil { + t.Error("auth router panicked; check constructor/handler wiring") + } + }() + response := call("POST", "/api/v1/auth/sessions", "", true) + if response.Code != 201 { + t.Fatal("Basic login route did not return 201") + } + var session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + } + if err := json.Unmarshal(response.Body.Bytes(), &session); err != nil || + session.Token == "" || session.TokenType != "Bearer" { + t.Fatal("login route returned invalid session") + } + other := f.session(t, user, password) + response = call("GET", "/api/v1/users/me", session.Token, false) + if response.Code != 200 || strings.Contains(response.Body.String(), "password") { + t.Fatal("Bearer user route failed or exposed password fields") + } + response = call("DELETE", "/api/v1/auth/sessions/current", session.Token, false) + if response.Code != 204 || response.Body.Len() != 0 { + t.Fatal("contract logout path must return empty 204") + } + for _, route := range [][2]string{ + {"GET", "/api/v1/users/me"}, {"DELETE", "/api/v1/auth/sessions/current"}, + } { + response = call(route[0], route[1], session.Token, false) + if response.Code != 401 || response.Header().Get("WWW-Authenticate") != "Bearer realm=\"messenger\"" { + t.Fatal("revoked token was accepted or challenged with wrong scheme") + } + } + if response := call("GET", "/api/v1/users/me", other.Token, false); response.Code != 200 { + t.Fatal("logout invalidated an independent session of the same user") + } + +} diff --git a/student-tests/lab4/CLEANUP.md b/student-tests/lab4/CLEANUP.md new file mode 100644 index 00000000..821a6b83 --- /dev/null +++ b/student-tests/lab4/CLEANUP.md @@ -0,0 +1,52 @@ +# Проверки уборки вложений + +Go-проверки используют управляемые ответы для граничного времени, пагинации с +пустой страницей и отказов LIST, EXISTS и DELETE. С переменной +`LAB4_TEST_DATABASE_URL` существующие тесты транзакции и сообщений дополнительно +используют выделенный PostgreSQL. + +```sh +go test -count=1 -v ./student-tests/auth_tests -run '^Test(AttachmentCleanup|AttachmentTx|MessageAttachments)' +``` + +Для сквозной проверки нужен отдельный Compose-проект с именем +`lab4-cleanup-<10 шестнадцатеричных символов>`, выделенными volumes и +синтетическими credentials от `scripts/init-local.sh`. Запустите его через +`up -d --build --wait`, задав отдельные loopback-порты через `LAB4_HTTP_PORT` +и `LAB4_POSTGRES_PORT`. Например, значение `0` поручает Docker выбрать порт. +Периодический сервис запускайте с `LAB4_CLEANUP_DRY_RUN=true`. +На отдельной копии исходников положите локальные credentials вне каталога +снимка и задайте отдельный `--project-directory` с путями к этим файлам. +При этом в тестовом override укажите абсолютные пути к build contexts, +`infra/haproxy.cfg` и `infra/s3/bootstrap.sh` из снимка. + +Передайте тесту локальный JSON с полями `project`, `runtime`, `snapshot` +и `base`. `runtime` указывает на каталог временных `.env` и `secrets/`, +`snapshot` на проверяемое дерево, а `base` содержит массив аргументов +`["docker", "compose", "-p", "", …]`, с которым поднят проект. +Файл спецификации содержит пути. Значения ключей в JSON не записываются. + +```sh +python3 student-tests/lab4/attachment_compensation.py --spec /tmp/cleanup-spec.json \ + --output student-tests/results/cleanup-run +``` + +Сценарий останавливает штатный dry-run сервис в выделенном проекте, +выполняет one-shot и поднимает отдельный периодический процесс на время теста. +Grace составляет 8 секунд. Пробные объекты, чаты и триггеры принадлежат +одноразовому проекту. После проверки удалите только этот проект и его volumes. + +Чтобы запустить тот же сценарий из существующего `run.py`, добавьте в +локальный конфиг `hooks.attachment_compensation` со значением +`["python3", "student-tests/lab4/attachment_compensation.py", "--spec", "/tmp/cleanup-spec.json", "--as-hook"]`. +Режим hook выдаёт наблюдения для существующего валидатора после всех проверок. +`--only killed_api_after_put` и `--only lost_response` запускают один сбой +и сохраняют его доказательства. + +Потеря ответа моделируется локальным посредником, получающим фактический +upstream 201 и закрывающим downstream без передачи HTTP-ответа. После SIGKILL +контейнера PostgreSQL может продолжать ждать тестовую advisory-блокировку. +Поэтому сначала фиксируются совпадение IP и контейнера, ожидание конкретного +PID, появление объекта в S3 и отсутствие строки. Затем проверяются события +Docker signal 9 и exit 137, снимается тестовый блокер и проверяется исчезновение +того же PID, отсутствие COMMIT и удаление объекта после grace. diff --git a/student-tests/lab4/README.md b/student-tests/lab4/README.md new file mode 100644 index 00000000..1ca15bdb --- /dev/null +++ b/student-tests/lab4/README.md @@ -0,0 +1,205 @@ +# Покрытие и воспроизводимость + +Подключённый S3 adapter и результаты на `highload-lab4:8081`: [S3.md](S3.md). +Старое описание bucket adapter ниже задаёт его интерфейс; теперь он +реализован в `s3_infrastructure.py`, а не остаётся пустой точкой расширения. + +## Источники требований + +Проверены `README.md`, `COURSE.md`, `CONTRIBUTING.md`, `MAINTAINERS.md`, +`contracts/README.md`, весь OpenAPI и `contracts/examples.json`, +`contracts/infrastructure.json`, `examples/requests.http`, `tests/README.md`, +`tests/fixtures/README.md`, публичные HTTP-клиент/сценарии, `infra/README.md`, +`infra/haproxy.cfg`, `.env.example`, `compose.example.yaml`, `compose.yaml`, +Dockerfile/API/migrate, Makefile, restart/stop hooks и их examples, +`evidence/README.md`, `REPORT.md` и `.github/pull_request_template.md`. +Каталога `infrastructure/` в этом checkout нет; требования инфраструктуры +находятся в `contracts/infrastructure.json` и `infra/`. +`.env` и значения secrets не читаются и не копируются в результаты. + +Не следует путать rubric lab4 с rubric lab3: новая задача lab4 — S3/вложения. +Идемпотентность, общие сессии, ≥2 API и LB — обязательные сохраняемые +возможности. Запрошенные lifecycle/лимиты/rolling проверки включены отдельно; +они не заменяют четыре новых пункта уровня 4 lab4. + +| Основание | Проверки | +| --- | --- | +| Обязательный унаследованный контракт | 400 для отсутствующего/неверного ключа; допустимые 1/128 ASCII и запрещённые символы; тот же 201 и полный Message; эквивалентный JSON с `attachment_ids: []`; 409 без записи; scope user/chat; одновременные повторы через ≥2 реплики; потерянный ответ, подтверждённый COMMIT с клиентским timeout; replay после restart/смены реплики; logout и чужой чат; live/ready | +| Повторная проверка ACL | До/после удаления членства исходного пользователя replay даёт 404, после восстановления возвращает исходный Message. Это дополнительная fault-injection проверка обязательного ACL: публичного API изменения участников нет | +| Запрошенные эксплуатационные сценарии | Каждая наблюдаемая API-реплика по очереди останавливается/возвращается; тот же токен; чтение/запись ≤10 с от начала stop; исключение ready=503 при live=200; SIGTERM и отказ нового соединения; enforced CPU/RAM всех контейнеров, закрытые API-порты, readiness/grace; rolling с реальной заменой обоих контейнеров; миграции один раз до старта API | +| Обязательный lab4, уровень 3 | Публичный smoke с SHA-256 и ACL, persistence всех stores и файла, failover вложения, private S3; собственные attachment-only message, побайтный download, чужой/неизвестный attachment и другой чат, pixel/byte limits, S3 outage | +| Обязательный lab4, уровень 4 | Oversize/corrupt/MIME/foreign references; компенсация разрыва и отказа S3/DB с grace; ≥10 параллельных upload с CPU/RAM; повтор bootstrap, отдельные ограниченные credentials и закрытая console/private bucket | +| Наше дополнительное качество | `rolling_update_zero_http_errors_quality`: строго ноль ошибок HTTP. Отдельный результат, не подмена буквальной рубрики | +| Необязательные рекомендации | Более долгий soak, k6/open-loop, несколько физических hosts, исследования на «5». Не включены в PASS этих сценариев | + +## Запуски и результаты + +```sh +export COMPOSE_PROJECT_NAME=highload-lab4 +python3 student-tests/lab4/run.py --config student-tests/lab4/config.example.json \ + --output student-tests/results/idempotency-01 --scenario idempotency +``` + +Другие значения `--scenario`: `restart`, `failover`, `grade4`, `rolling`, +`attachments`, `public`, `all`. Полный запуск: `student-tests/verify.py`. +Конфиг — JSON без секретов: `base_url`, `ca_file`, `concurrency` (2..64 для race), +`request_timeout_seconds`, `failover_window_seconds` (11..60), +`rolling_window_seconds` (минимальная длительность), `rolling_max_seconds` +(60..300, верхняя граница всего update), `compose_base_argv`, +`compose_config_argv`, `api_services`, `lb_services`, `restart_services`, `hooks`. +Это Docker Compose runner, не Swarm runner. Rolling рассчитан на две API. +Для другого Compose проекта установите `COMPOSE_PROJECT_NAME`/`COMPOSE_FILE` +во внешнем окружении; все Docker и public subprocess наследуют их. + +`events.jsonl` содержит каждую операцию GET/POST нагрузки, monotonic время от +начала воздействия, latency, успех/ошибку, instance и ID подтверждения, +hook timings, Docker IDs/StartedAt/volumes, агрегаты и статусы. Пароли, токены, +Authorization, signed URL и bodies регистрации не сохраняются. `summary.json` +содержит статусы каждого case; stdout/stderr каждой публичной make-команды +сохраняются дословно в `*.stdout.txt`/`*.stderr.txt`. + +Нагрузка — последовательный closed-loop POST+GET; race отдельно запускает +`concurrency` одновременных POST одного ключа. Это измерение доступности, +не максимальной производительности. RPS, ошибки и error fraction считаются +по реальным попыткам. p50/p95/p99 — nearest-rank только успешных POST. +При неопределённом ответе повторяется тот же ключ/body. После нагрузки история +проверяется на сохранность подтверждений и отсутствие дублей всех попыток. +Stop работает параллельно с нагрузкой; recovery считается от начала hook, +первый успешный запрос должен быть начат после окончания stop и закончиться +≤10 с. Это консервативная верхняя оценка с интервалом опроса клиента. +Rolling нагружается до завершения hook, затем проверяются ID, шаги замены и +история. Эксперимент на одном Docker-host не доказывает HA PostgreSQL/LB/host. + +`public_lb_endpoint` сверяет порт URL с опубликованным портом выбранного LB +и запрещает прямые host bindings API. DNS/соответствие hostname выбранному +Docker-host оператор фиксирует при настройке; совпадение порта само по себе +не доказывает удалённую DNS-топологию. С одним токеном должны наблюдаться все +работающие API-контейнеры; каждый остановленный alias проверяется Docker inspect. +Пока LB не работает или есть только одна API, это не PASS. Текущий checkout +уже объявляет `api.scale: 2` и `lb`; это декларация, не runtime-доказательство. + +Restart сравнивает все API, PostgreSQL, Redis, S3 и LB до/после: прежние IDs и +volumes, новый StartedAt и running. Migrator должен сохранить StartedAt. +После restart прежний токен и тот же ключ обязаны вернуть исходный Message. +Публичный persistence дополнительно проверяет оригинал файла. Проверки не +выжидают 24 часа и не доказывают полный срок удержания ключа этим ожиданием. + +## Hooks: готовые сценарии, ожидающие инфраструктуры + +Hooks — argv-массивы без shell interpolation. Отсутствующий hook даёт +`PENDING_INFRA`; фиктивные команды, возвращающие 0, не заменяют доказательства. +`config.example.json` подключает существующие restart/stop и собственные +start/recover адаптеры. Последние только запускают уже существующие контейнеры +API выбранного проекта и проверяют Docker health, не создавая инфраструктуру. +Оператору нужны запущенный Docker и готовый синтетический стенд. + +Перед fault проверяется наличие restoration hooks. `TARGET_INSTANCE_ID` — +alias целевого процесса; stdout/stderr hook скрыты, кроме описанного JSON. +Не применять SQL/network fault adapters к рабочей БД: только одноразовый +учебный проект и точно выбранные UUID. Не использовать `down -v`, flush или seed. + +| Hook | Действие и возвращаемое наблюдение | +| --- | --- | +| `stop_one` / `start_one` | Остановить ровно выбранный API за ≤10 с и оставить остановленным; вернуть тот же контейнер/volume. Stop stdout не нужен | +| `restart` | Реально перезапустить API и все stores, сохранив volumes; миграции не повторять. ≤90 с | +| `recover_all` | Вернуть обе API после public failover. Пример подключён в config | +| `make_unready` / `restore_ready` | Нарушить только доступ выбранного API к критичной зависимости, сохранив доступ LB к API; восстановить. Нужна подходящая сеть/fault adapter, сейчас PENDING | +| `probe_health` | Внутренний GET выбранного API: `{"live_status":200,"ready_status":503}`. Private probe не публикует host-порт API | +| `lb_timing_evidence` | По действующей конфигурации: `health_interval_seconds`, `health_timeout_seconds`, `configured_max_detection_seconds` — положительные числа; detection ≤10 | +| `sigterm_one` / `probe_stopped` | Послать SIGTERM без принудительного SIGKILL, подтвердить остановку/новое TCP-соединение. Probe: `{"accepting":false,"signal":"SIGTERM","forced_kill":false,"exit_code":0,"connection_probe_performed":true}` | +| `rolling_update` | Поочерёдно заменить обе API при ≥1 ready, stdout JSON по формату ниже. Самого механизма rolling в текущем checkout нет: PENDING | +| `migration_evidence` | Захватить lifecycle до первого старта проекта, вернуть исходные события по формату ниже. `compose ps`/RestartCount не доказывают однократность | +| `revoke_access` / `restore_access` | В изолированной синтетической фикстуре временно убрать членство и вернуть его; вход `TEST_CHAT_ID`, `TEST_USER_ID`, `TEST_REPLACEMENT_USER_ID`, `TEST_MESSAGE_ID`, `TEST_IDEMPOTENCY_KEY`. Учитывать FK sender→member. Вернуть `{"action":"revoke_access","validated_exact_message":true}` или restore. Не нужен новый публичный API | + +Rolling JSON: `replaced_processes >= 2`, `min_ready_during_update >= 1`, +`steps` для обоих реальных старых/новых ID. Шаг: +`{"old_id":"...","new_id":"...","begin_ns":1,"end_ns":2,"ready_survivor_ids":["..."]}`. +Шаги не перекрываются; другой готовый backend подтверждён на каждом шаге. +Нужны реальные timestamps и IDs, не константы из этого описания. + +Migration JSON: `capture_started_before_stack: true`, `events` — реальные +Docker lifecycle события `{service, action, container_id, time_ns, exit_code?}`. +Ровно один `migrate/start`, один `migrate/die` с exit_code=0, ≥2 разных +`api/start`, все после успешного `die`. Capture должен покрывать также rolling, +чтобы повторный migrator не пропал из доказательства. Имена в событиях — роли +`api`/`migrate`, адаптер сопоставляет им реальные services. + +## Специальные lab4 adapters + +Эти готовые точки запуска выполняют реализационно-зависимые сценарии и +проверяют наблюдения. Пока соответствующие hooks не подключены, результат +`PENDING_INFRA`. JSON adapter должен быть обезличенным, без credentials, +URL объектов или содержимого S3. Он не должен просто печатать ожидаемые поля: +оператор реализует воздействие и измерение, сохраняет первичные доказательства +и описывает команды в REPORT. Runner проверяет их структуру/критерии; истинность +внутренних воздействий требует ревью адаптера и защиты, как у публичных hooks. + +| Hook | Обязательные наблюдения | +| --- | --- | +| `attachment_compensation` | `injected_faults` включает `client_disconnect`, `after_s3_before_db`; `grace_seconds > 0`, `untracked_objects_after_grace = 0`, `fresh_objects_preserved = true`, `referenced_objects_preserved = true`. Документировать механизм и cutoff; проверить исходные объекты до/после | +| `attachment_parallel_memory` | `concurrent_uploads >= 10`, `successful_uploads >= 10`, `body_bytes > 0`, `duration_seconds > 0`, `memory_limit_bytes`, `oom_kills = 0`, исходные `samples: [{time_seconds,memory_bytes,cpu_percent}]`. Пик RAM ≤ лимита; указать sampling interval и ограничения. Только константа peak без samples не принимается | +| `attachment_bucket_security` | `bootstrap_runs >= 2`, `bucket_survived_second_bootstrap = true`; unauthenticated direct GET и LIST (`anonymous_get_status`, `anonymous_list_status`) дают 401/403/404; `api_credentials_are_admin = false`, `api_can_administer_other_buckets = false`, `console_reachable_from_shared_network = false` | +| `attachment_s3_failure` | `fault = s3_unreachable`, `upload_status = 503`, `ready_status = 503`, `live_status = 200`, `false_ready_attachments = 0`, `restored_original_checksum_equal = true` | +| `attachment_pixels_limit` | Валидная картинка: `width*height > 40000000`, `0 < encoded_bytes <= 10485760`, `upload_status = 413`; fixture генерируется временно, не коммитится большим файлом | + +Базовые negative upload и ACL не требуют этих hooks и запускаются как обычные +HTTP cases. Проверка private S3 обращается к внутреннему S3 только из адаптера; +весь пользовательский API-трафик идёт через публичный LB. + +## Транзакция Message + attachments до подключения S3 + +`student-tests/auth_tests/message_attachments_test.go` дополняет тот же набор +Go-тестов и переиспользует общую PostgreSQL-фикстуру из `storage_test.go`. +Новая переменная `LAB4_TEST_DATABASE_URL` указывает на выделенную учебную БД +с правом CREATE SCHEMA. Redis, S3, сетевой HTTP-сервер и работающий API не нужны. +Создаётся уникальная схема, в ней применяются существующие миграции; по +завершении удаляется только эта схема. Без переменной каждый case даёт явный +`SKIP: PENDING_INFRA`. Если DSN задан, но БД недоступна, это FAIL, а не PASS. + +Проверки вызывают настоящий MessageHandler → MessageService → PostgreSQL +repository через `httptest.ResponseRecorder`. Текущий пользователь подставляется +в request context: аутентификация проверяется прежними auth-тестами. Вложения +заранее создаются как **метаданные**, без настоящих объектов S3. Такой прогон +проверяет транзакцию/ACL/историю, но не подтверждает upload/download. + +```sh +# DSN подготовить в локальном окружении, не публиковать credentials. +GOCACHE=/private/tmp/highload-lab4-go-cache go test -count=1 -json \ + ./student-tests/auth_tests -run '^TestMessageAttachments' \ + > student-tests/results/message-attachments.jsonl +``` + +| TestMessageAttachments… | Проверяемое поведение | +| --- | --- | +| SingleIDServiceRegression | Без БД: настоящий MessageService создаёт сообщение с одним ID без panic; первый и повторный вызов с одинаковыми значениями и UUID в верхнем регистре проходят до репозитория с сохранением UUID-идентичности. Подставной repository не доказывает SQL-idempotency — это проверяют группы ниже | +| TextCompatibility | Прежний текстовый JSON без attachment_ids, явный [] на replay: тот же 201/Message, одна строка истории, ноль связей | +| OnlyAndOrderZeroToTen | Для каждого количества 0..10 — точный список/порядок в ответе, истории и SQL position=0..n−1. При n>0 текст пуст; другой участник прикрепляет доступные ему файлы чата. Для n=1 первый и повторный POST используют одинаковый JSON с верхним регистром UUID: оба 201, тот же Message, снимок БД не меняется | +| InvalidReferencesAreAtomic | Валидный первый ID + неизвестный/ID другого чата; чужой чат. 404, полный снимок messages/links/idempotency/metadata не меняется, ключ можно использовать после исправления запроса | +| InvalidListLeavesNoRows | Дубликаты и 11 ID: 400 без изменений БД | +| IdempotencyIncludesListAndOrder | Первый body уже содержит UUID заглавными буквами; повторяется тот же body: прежний 201/Message без изменений. Другой порядок/список/количество/text: 409, полный снимок БД неизменен | +| ConcurrentReplayDoesNotMultiplyLinks | Восемь одновременных запросов одного ключа: все 201 и один Message, ровно одна запись ключа и три связи, без дублей истории | +| LinkFailureRollsBackEverything | Test-only trigger в уникальной схеме падает при второй связи после видимой первой. Неоткатываемая sequence подтверждает достижение fault; сообщение, связи, ключ и metadata полностью откатываются. После снятия fault тот же ключ успешно создаёт ровно одно полное сообщение | + +В `lab4/run.py` расширен существующий `attachment_acl_bytes_and_idempotency`, +а не создан второй HTTP-набор: 0..10 ID в порядке запроса, attachment-only, +404 без изменения истории, replay и 409 для изменённого порядка/списка, +конкурентный повтор. Этот сквозной case требует уже работающего upload и LB; +число SQL-связей и настоящий rollback доказываются PostgreSQL-проверками выше. + +Проверка регистра не объявляет изменение строки UUID между запросами +эквивалентным body. Первый запрос **сразу** использует верхний регистр, +повтор использует точно тот же JSON/ключ. Обязательные оговорки контракта про +порядок ключей JSON и отсутствие `attachment_ids` вместо `[]` сохранены; +новое правило равенства разных строк запроса не вводится. Порядок и количество +ID в ответе проверяются с учётом UUID-идентичности: сервер вправе вернуть +каноническое написание. Сами Message первого ответа и replay сравниваются полностью. + +Только сервисная регрессия, без БД/Redis/S3/LB: + +```sh +GOCACHE=/private/tmp/highload-lab4-go-cache go test -count=1 -v \ + ./student-tests/auth_tests -run '^TestMessageAttachmentsSingleIDServiceRegression$' +``` + +Итоговый прогон транзакций, файлового API и унаследованных сценариев +приведён в [REPORT.md](../../REPORT.md). diff --git a/student-tests/lab4/S3.md b/student-tests/lab4/S3.md new file mode 100644 index 00000000..cc8ccf01 --- /dev/null +++ b/student-tests/lab4/S3.md @@ -0,0 +1,73 @@ +# Автоматические проверки S3-инфраструктуры + +Запуск из HighLoad на разрешённом учебном проекте: + +```sh +python3 student-tests/lab4/s3_infrastructure.py \ + --project highload-lab4 --base-url http://127.0.0.1:8081 \ + --output student-tests/results/s3-run-01 +``` + +Нужен Docker с доступом к выбранному проекту, его `s3`, `s3-init`, две API и LB. +Поддержан `highload-lab4` или явно выбранный изолированный `lab4-s3-test-*`. +Каталог результатов должен быть новым. Программа **не создаёт стенд**, +не меняет Compose/Go/миграции, не удаляет volumes. Она работает с текущим +`s3-init` и установленным `rustfs/rc`, использует секреты внутри одноразовых +клиентских контейнеров, а не читает/печатает их в Python на host. + +Это реализация уже предусмотренного `hooks.attachment_bucket_security`, +подключённая в `config.example.json` через `--as-hook`. Отдельного альтернативного +набора проверок нет: standalone и общий lab4 runner вызывают один код. +Общий runner передаёт каталог `STUDENT_TEST_EVIDENCE_DIR`, где сохраняется +дочерний summary/events. В hook mode stdout содержит только итоговый JSON +наблюдений для прежнего валидатора, и только после успеха всех cases. +Код 0 — PASS, 1 — FAIL, 2 — PENDING_INFRA; недоступный Docker не становится PASS. + +Конфигурация собственных проверок теперь указывает **8081 и highload-lab4**, +чтобы случайно не проверить работающую lab3 на 8080. Для другого проекта +поменяйте также `compose_base_argv`, `compose_config_argv` и аргументы hook, +а не только URL. Рабочий Compose при этом не редактируется тестом. + +| Сценарий | Наблюдение / основание | +| --- | --- | +| Реальный API Put/Get/List/Delete | API-ключ записывает синтетические бинарные bytes в `messenger-attachments`; скачивание сравнивается `cmp` и SHA-256; LIST находит объект; после DELETE успешный LIST его не содержит | +| Bootstrap дважды | Уже записанный объект переживает **два реальных выполнения** `/bootstrap.sh` через одноразовый `s3-init`. RustFS ID/StartedAt/named volume не меняются. Критерий lab4/4 | +| Restart RustFS | Перезапускается только `s3`, без `up/down` и пересоздания stores. ID/volume прежние, StartedAt новый; скачанные bytes совпадают. Сохранность S3 — часть lab4/3 | +| Private bucket вне API | Unsigned GET **существующего** объекта и unsigned LIST дают строго HTTP 403. Предшествующий signed GET подтверждает, что ресурс есть. Private bucket — lab4/3 и доказательство на 4 | +| Права в другом bucket | Root подготавливает новый временный bucket и известный sentinel. API Put/Get/Delete/List должны дать AccessDenied, а не произвольную сетевую ошибку/NoSuchBucket. Критерий lab4/4 | +| CreateBucket и IAM/admin | Создание нового bucket ключом API и IAM user/policy list отклоняются. Дополнительно root читает **реальную** привязку пользователя: ровно `messenger-attachments-api-v1`, без групп; документ разрешает только пять S3-действий ниже. IAM/admin grants отсутствуют. Критерий lab4/4 | +| Console | Отдельный клиент в общей сети успешно получает health:9000=200, но соединение к :9001 отклонено. DNS/недоступность всей сети не засчитывается как закрытая консоль. Критерий lab4/4 | +| API ×2 за HAProxy | Docker inspect подтверждает ≥2 healthy API без host ports; URL принадлежит опубликованному LB. До/после restart одним и тем же Bearer наблюдаются все реальные X-Instance-Id. Сохраняемая возможность lab3 | +| Cleanup | Удаляются только созданные тестом objects и два bucket с UUID в имени. У bucket приложения удаляется только точный тестовый key; рекурсивного удаления bucket, reset IAM, flush и удаления volumes нет | + +Разрешения runtime policy проверяются по точным парам Action/Resource: + +- `s3:ListBucket`, `s3:GetBucketLocation` → `arn:aws:s3:::messenger-attachments`; +- `s3:PutObject`, `s3:GetObject`, `s3:DeleteObject` → тот же bucket с `/*`. + +Wildcard для других buckets/IAM, дополнительная policy или унаследованная +группа не принимаются. Успешные операции дополнительно проверяют, что это +не просто декларация прав при неработающем ключе. Проверяется текущая +конкретная bootstrap policy; это не общий аудит всех API RustFS. + +`events.jsonl` хранит исходные exit codes, факт AccessDenied, HTTP statuses, +cmp/SHA-256/размер, реальные Docker IDs/StartedAt/volumes и время готовности. +Сохраняются также имена **сгенерированных тестом** namespaces для разбора cleanup. +Исходные ответы `rc` не сохраняются: IAM user info и ошибки могут содержать +access key. В evidence выводятся только проверенные grants и обезличенные +наблюдения. Сессионные токены остаются в памяти публичного клиента. Синтетический +пользователь для проверки LB остаётся в PostgreSQL: публичный API не имеет +удаления пользователей; его данные тест не удаляет через обходной SQL. + +Проверено 11.11.2026 на `highload-lab4`: RustFS 1.0.4, rc v0.1.30, HAProxy 3.2.24. +Основной прогон — 10 PASS в +[итоговое подтверждение приватности и прав](../../evidence/lab4-final/s3-security.json). +Дополнительная read-only проверка реальной policy, уже включённая в общий +код, — PASS в `effective-policy/summary.json`. Все 19 проверок runner — PASS. +Дополнительный запрос policy не перезапускал S3 повторно. + +Прогон проверяет инфраструктуру и прямой S3 I/O. Go upload/download, +ACL вложений через API, компенсация БД/S3 и нагрузка из ≥10 upload здесь +не проверяются и не объявляются пройденными. Оценка всей lab4 автоматически +не подтверждается. Ручной исходный протокол: +[отчёте по лабораторной](../../REPORT.md). diff --git a/student-tests/lab4/attachment_compensation.py b/student-tests/lab4/attachment_compensation.py new file mode 100644 index 00000000..18bd421b --- /dev/null +++ b/student-tests/lab4/attachment_compensation.py @@ -0,0 +1,428 @@ +"""Section 2.4 runtime checks for a disposable lab4-cleanup-* Compose project.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import threading +import time +from datetime import datetime, timezone +from urllib.parse import quote, urlsplit +import uuid +import xml.etree.ElementTree as ET + +from run import Client, Evidence, Failure, check + +BUCKET = "messenger-attachments" +GRACE = 8 +LOCK = 9031427 + + +class Experiment: + def __init__(self, spec, evidence): + check(re.fullmatch(r"lab4-(?:cleanup-[a-f0-9]{10}|final-[a-z0-9-]+)", spec["project"]), "requires an isolated cleanup project") + check(spec["base"][:4] == ["docker", "compose", "-p", spec["project"]], "invalid project command") + self.spec, self.base, self.e = spec, spec["base"], evidence + self.runtime = Path(spec["runtime"]) + path = Path(spec["snapshot"]) / "tests/fixtures/sample.png" + self.png = path.read_bytes() + self.sha = hashlib.sha256(self.png).hexdigest() + self.c = Client("http://" + self.port("lb", 8080), timeout=8) + self.nonce = uuid.uuid4().hex[:10] + self.flags = dict(os.environ, LAB4_HTTP_PORT="0", LAB4_POSTGRES_PORT="0") + + def run(self, args, data=None, allowed=(0,), timeout=28): + result = subprocess.run(args, input=data, capture_output=True, timeout=timeout, env=self.flags if hasattr(self, "flags") else None) + check(result.returncode in allowed, "command failed: " + " ".join(args[:4]) + " exit " + str(result.returncode)) + return result + + def port(self, service, internal): + binding = self.run(self.base + ["port", service, str(internal)]).stdout.decode().strip() + check(re.fullmatch(r"127\.0\.0\.1:\d+", binding), "expected loopback binding") + return binding + + def sql(self, query): + return self.run(self.base + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-At", "-v", "ON_ERROR_STOP=1", "-c", query]).stdout.decode().strip() + + def s3(self, method, key="", query="", body=None): + # Credentials travel over stdin to curl in the isolated S3 container. + access = (self.runtime / "secrets/s3_access_key_id").read_text().strip() + secret = (self.runtime / "secrets/s3_secret_access_key").read_text().strip() + config = f'user = "{access}:{secret}"\n' + args = self.base + ["exec", "-T", "s3", "curl", "--config", "-", "-fsS", "--max-time", "8", + "--aws-sigv4", "aws:amz:us-east-1:s3", "-X", method] + if body is not None: + check(body == b"fixture", "unexpected synthetic body") + args += ["--data-binary", "fixture"] + url = "http://127.0.0.1:9000/" + BUCKET + ("/" + quote(key, safe="/") if key else "") + query + return self.run(args + [url], config.encode()).stdout + + def objects(self): + xml = ET.fromstring(self.s3("GET", query="?list-type=2")) + check(xml.findtext("{*}IsTruncated") in ("false", "False"), "unexpectedly large runtime fixture") + return {item.findtext("{*}Key"): datetime.fromisoformat(item.findtext("{*}LastModified").replace("Z", "+00:00")) + for item in xml.findall("{*}Contents")} + + def put(self, key): + self.s3("PUT", key, body=b"fixture") + + def key(self, label): + return "attachments/cleanup-" + self.nonce + "-" + label + + def await_old(self, key, timeout=23): + self.wait(lambda: self.age(key) > GRACE + 1, "object never crossed grace period", timeout) + + def age(self, key): + date = self.objects().get(key) + check(date is not None, "object absent before age check") + return (datetime.now(timezone.utc) - date).total_seconds() + + @staticmethod + def wait(test, message, timeout=12): + deadline = time.monotonic() + timeout + while True: + result = test() + if result: + return result + check(time.monotonic() < deadline, message) + time.sleep(.15) + + def once(self, dry=False, grace=GRACE, exit_code=0): + argv = self.base + ["run", "--rm", "--no-deps", "-T", "attachment-cleanup", "/app/attachment-cleanup", "-once", + "-dry-run=" + str(dry).lower(), "-grace-period=" + str(grace) + "s", "-pass-timeout=12s"] + proc = self.run(argv, allowed=(0,1), timeout=24) + self.e.record("one_shot", dry_run=dry, grace_seconds=grace, exit_code=proc.returncode) + check(proc.returncode == exit_code, "incorrect cleanup command exit code") + + def setup(self): + self.run(self.base + ["stop", "attachment-cleanup"]) + a,b,e,c = self.c.bootstrap() + self.owner,self.member,self.outsider,self.chat = a,b,e,c + self.e.record("sql_isolation", value=self.sql("SHOW transaction_isolation")) + check(self.sql("SHOW transaction_isolation") == "read committed", "unexpected postgres isolation") + + def upload(self, filename="fixture.png", status=201): + return self.c.call("POST", f"/api/v1/chats/{self.chat['id']}/attachments?filename={filename}", status, + actor=self.owner, raw=self.png, headers={"Content-Type":"image/png"}, + schema="Attachment" if status == 201 else None) + + def key_for(self, attachment): + aid = str(uuid.UUID(attachment["id"])) + return self.sql(f"SELECT object_key FROM attachments WHERE id='{aid}'") + + def count(self, where): + return int(self.sql("SELECT count(*) FROM attachments WHERE " + where)) + + def count_upload(self, filename): + chat_id = str(uuid.UUID(self.chat["id"])) + check(re.fullmatch(r"[a-z-]+\.png", filename), "unexpected fixture filename") + return self.count(f"chat_id='{chat_id}' AND filename='{filename}'") + + def get(self, attachment): + path = "/api/v1/attachments/" + attachment["id"] + check(self.c.call("GET", path, actor=self.member, schema="Attachment")["json"] == attachment, "metadata changed") + body = self.c.call("GET", path + "/content", actor=self.member)["bytes"] + check(body == self.png and hashlib.sha256(body).hexdigest() == self.sha, "original changed") + self.c.call("GET", path + "/content", 404, actor=self.outsider) + + def hold_sql(self, statement, lock_visible): + p = subprocess.Popen(self.base + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + p.stdin.write((statement + "\n").encode()) + p.stdin.flush() + self.wait(lock_visible, "SQL holder did not acquire lock") + return p + except BaseException: + p.stdin.close() + p.wait(timeout=8) + raise + + @staticmethod + def end_holder(proc, sql="ROLLBACK;"): + proc.stdin.write((sql + "\n").encode()) + proc.stdin.close() + check(proc.wait(timeout=8) == 0, "lock transaction failed to close") + + def retention(self): + orphan = self.key("orphan") + outside = "other/cleanup-" + self.nonce + busy_commit = self.key("busy-commit") + busy_rollback = self.key("busy-rollback") + fresh = self.key("fresh") + first = self.upload("without-message.png")["json"] + linked = self.upload("with-message.png")["json"] + self.c.message(self.owner, self.chat, "", attachments=[linked["id"]]) + referenced = [self.key_for(first), self.key_for(linked)] + for key in [orphan, outside, busy_commit, busy_rollback]: + self.put(key) + self.await_old(busy_rollback) + before = self.objects() + for key in [orphan, *referenced, busy_commit, busy_rollback]: + check((datetime.now(timezone.utc)-before[key]).total_seconds() > GRACE, "expected old fixture") + self.once(dry=True) + check(self.objects() == before, "dry run changed S3") + user, chat = self.owner["id"], self.chat["id"] + values = f"('{chat}','{user}','held.png','{busy_commit}','image/png',{len(self.png)},'{self.sha}')" + holder = self.hold_sql(f"""BEGIN; + SELECT pg_advisory_xact_lock(hashtextextended('{busy_commit}',0)); + INSERT INTO attachments(chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES {values}; + SELECT pg_advisory_xact_lock({LOCK});""", + lambda: self.sql(f"SELECT NOT pg_try_advisory_xact_lock({LOCK})")=="t") + try: + rollback = self.hold_sql(f"""BEGIN; + SELECT pg_advisory_xact_lock(hashtextextended('{busy_rollback}',0)); + SELECT pg_advisory_xact_lock({LOCK + 1});""", + lambda: self.sql(f"SELECT NOT pg_try_advisory_xact_lock({LOCK + 1})")=="t") + try: + check(self.count(f"object_key='{busy_commit}'")==0, "held row visible before COMMIT") + self.put(fresh) + check(self.age(fresh)pg_backend_pid())")=="t", + "cleanup was not observed waiting in EXISTS",8) + self.run(["docker","stop","--time","5",name]) + state=json.loads(self.run(["docker","inspect",name]).stdout)[0]["State"] + check(not state["Running"] and state["ExitCode"]==0, "SIGTERM did not terminate cleanly") + check(self.sql("SELECT EXISTS(SELECT 1 FROM pg_locks WHERE locktype='advisory' AND NOT (classid=0 AND objid="+str(LOCK)+"))")=="f", + "cleanup session retained its advisory lock") + check(waiting in self.objects(),"SIGTERM deleted object without EXISTS") + finally: + self.end_holder(holder) + self.run(["docker","start",name]) + self.wait(lambda:waiting not in self.objects(),"restart failed to reconcile",12) + self.e.record("sigterm", during="EXISTS", exit_code=0, key=waiting, lock_released=True, restart_deleted=True) + finally: + self.run(["docker","rm","-f",name],allowed=(0,1)) + + +def main(): + p=argparse.ArgumentParser(description=__doc__) + p.add_argument("--spec",required=True,help="private temporary Compose specification") + p.add_argument("--output",default=os.environ.get("STUDENT_TEST_EVIDENCE_DIR")) + p.add_argument("--as-hook",action="store_true") + p.add_argument("--only", choices=["killed_api_after_put", "lost_response"]) + args=p.parse_args() + if not args.output: + p.error("--output is required") + import contextlib + import sys + with contextlib.redirect_stdout(sys.stderr if args.as_hook else sys.stdout): + e=Evidence(args.output) + x=Experiment(json.loads(Path(args.spec).read_text()),e) + e.case("setup",x.setup) + if e.summary["setup"]["status"]=="PASS": + for name,job in [("retention",x.retention),("fault_after_put",x.rejected_insert), + ("aborted_body",x.aborted_body),("killed_api_after_put",x.killed_api), + ("lost_response",x.lost_response),("periodic_sigterm",x.periodic)]: + if not args.only or name == args.only: + e.case(name,job) + code=e.finish() + if args.as_hook and code==0 and not args.only: + rows=[json.loads(line) for line in e.path.read_text().splitlines()] + observations={value["event"]:value for value in rows} + print(json.dumps({ + "injected_faults":[name for name in ("client_disconnect","after_s3_before_db") if name in observations], + "grace_seconds":observations["one_shot"]["grace_seconds"], + "untracked_objects_after_grace":int(not observations["after_s3_before_db"]["deleted_after_grace"]), + "fresh_objects_preserved":observations["retention"]["fresh_preserved"], + "referenced_objects_preserved":observations["retention"]["referenced_preserved"], + })) + return code + + +if __name__=="__main__": + raise SystemExit(main()) diff --git a/student-tests/lab4/attachment_pixel_limit.py b/student-tests/lab4/attachment_pixel_limit.py new file mode 100644 index 00000000..1f7b06a0 --- /dev/null +++ b/student-tests/lab4/attachment_pixel_limit.py @@ -0,0 +1,26 @@ +import argparse +import json +import struct +import sys +import zlib +from pathlib import Path +sys.path.insert(0,str(Path.cwd()/"tests")) +from client import Client +parser = argparse.ArgumentParser(description="Check the decoded pixel limit") +parser.add_argument("--base-url", required=True) +parser.add_argument("--output", type=Path, required=True) +args = parser.parse_args() +p = args.output +p.mkdir(parents=True, exist_ok=True) +def chunk(name,data): + return struct.pack(">I",len(data))+name+data+struct.pack(">I",zlib.crc32(name+data)) +w,h=6500,6200 +obj=zlib.compressobj() +row=bytes(w+1) +blob=b"".join(obj.compress(row) for _ in range(h))+obj.flush() +data=b"\x89PNG\r\n\x1a\n"+chunk(b"IHDR",struct.pack(">IIBBBBB",w,h,8,0,0,0,0))+chunk(b"IDAT",blob)+chunk(b"IEND",b"") +c=Client(args.base_url) +alice,_,_,chat=c.bootstrap() +response=c.call("POST","/api/v1/chats/"+chat["id"]+"/attachments?filename=large.png",413,actor=alice,raw=data,headers={"Content-Type":"image/png"}) +assert response["json"]["error"]["code"]=="payload_too_large" +(p/"pixel-limit.json").write_text(json.dumps({"width":w,"height":h,"encoded_bytes":len(data),"upload_status":response["status"]},indent=2)+"\n") diff --git a/student-tests/lab4/attachment_runtime_faults.py b/student-tests/lab4/attachment_runtime_faults.py new file mode 100644 index 00000000..f778488d --- /dev/null +++ b/student-tests/lab4/attachment_runtime_faults.py @@ -0,0 +1,89 @@ +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +sys.path.insert(0, str(Path.cwd() / "tests")) +from client import Client, check +parser = argparse.ArgumentParser(description="Check attachment failover and S3 outage on a disposable stack") +parser.add_argument("--base-url", required=True) +parser.add_argument("--output", type=Path, required=True) +parser.add_argument("--probe", type=Path, required=True, help="Path to the Linux direct HTTP probe") +parser.add_argument("--probe-image", required=True, help="Locally built API image used to run the probe") +args = parser.parse_args() +project = os.environ["COMPOSE_PROJECT_NAME"] +if not project.startswith("lab4-final-"): + parser.error("use an isolated lab4-final-* project") +c = Client(args.base_url, timeout=3) +compose = ["docker", "compose", "--env-file", ".env"] +out = args.output +out.mkdir(parents=True, exist_ok=True) + +def run(*argv, **kw): + return subprocess.run(argv, check=True, capture_output=True, timeout=65, **kw).stdout + +def sql(query): + return run(*compose, "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-Atqc", query).decode().strip() + +def ready(): + end = time.monotonic()+30 + while time.monotonic()= 2, "concurrent requests did not traverse two LB backends") + case("idempotency_concurrent_cross_replica", concurrent) + + def auth_rechecked(): + key, text = unique("auth-key"), unique("auth") + post(client, alice, chat, {"text": text}, key) + client.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + post(client, alice, chat, {"text": text}, key, 401) + post(client, eve, chat, {"text": text}, key, 404) + check(len(occurrences(client, bob, chat, text)) == 1, "auth retry modified message") + case("idempotency_session_and_foreign_access", auth_rechecked) + + # The fixture owns one message in this chat. The isolated adapter temporarily + # assigns it to the other member so the FK permits removing the original sender. + def access_rechecked(): + if not all(config.get("hooks", {}).get(name) for name in ("revoke_access", "restore_access")): + raise Pending("configure isolated hooks.revoke_access/restore_access for synthetic fixture") + client2, owner, member, outsider, target, _ = bootstrap(config) + key, text = unique("acl-key"), unique("acl") + first = post(client2, owner, target, {"text": text}, key)["json"] + evidence.record("access_hook_target", chat_id=target["id"], user_id=owner["id"], + replacement_user_id=member["id"], message_id=first["id"]) + env = config.setdefault("hook_context", {}) + env.update({"TEST_CHAT_ID": target["id"], "TEST_USER_ID": owner["id"], + "TEST_REPLACEMENT_USER_ID": member["id"], "TEST_MESSAGE_ID": first["id"], + "TEST_IDEMPOTENCY_KEY": key}) + try: + hook(config, "revoke_access", evidence) + post(client2, owner, target, {"text": text}, key, 404) + finally: + hook(config, "restore_access", evidence) + check(post(client2, owner, target, {"text": text}, key)["json"] == first, + "replay after access restoration returned a different Message") + check(len(occurrences(client2, member, target, text)) == 1, + "access restoration left duplicate message") + case("idempotency_access_rechecked_after_membership_change", access_rechecked) + + def replay_after_restart(): + client3, owner, member, outsider, target, _ = bootstrap(config) + key, text = unique("restart-key"), unique("restart") + first = post(client3, owner, target, {"text": text}, key)["json"] + before = lifecycle_snapshot(config) + hook(config, "restart", evidence, timeout=90) + deadline = time.monotonic() + 90 + while True: + try: + client3.call("GET", "/health/ready", schema="Health") + break + except Failure: + check(time.monotonic() < deadline, "LB did not recover after restart") + time.sleep(.2) + after = lifecycle_snapshot(config) + verify_restart(before, after, evidence) + check(post(client3, owner, target, {"text": text}, key)["json"] == first, + "idempotency result changed after restart") + check(len(occurrences(client3, member, target, text)) == 1, + "restart retry created duplicate") + case("idempotency_survives_restart", replay_after_restart) + + def uncertain_replay(): + client4, owner, member, outsider, target, _ = bootstrap(config) + key, text = unique("uncertain-key"), unique("uncertain") + url = urlsplit(client4.base) + check(url.scheme in ("http", "https") and url.hostname, "invalid LB URL") + path = message_path(target) + body = json.dumps({"text": text}, ensure_ascii=False).encode("utf-8") + request = ("POST " + path + " HTTP/1.1\r\nHost: " + url.netloc + + "\r\nConnection: close\r\nAuthorization: Bearer " + owner["token"] + + "\r\nIdempotency-Key: " + key + + "\r\nContent-Type: application/json\r\nContent-Length: " + str(len(body)) + + "\r\n\r\n").encode("ascii") + body + sock = socket.create_connection((url.hostname, url.port or (443 if url.scheme == "https" else 80)), 3) + try: + if url.scheme == "https": + context = ssl.create_default_context(cafile=config.get("ca_file")) + sock = context.wrap_socket(sock, server_hostname=url.hostname) + sock.sendall(request) + # Deliberately discard the response. The first request may or may not commit. + finally: + sock.close() + result = post(client4, owner, target, {"text": text}, key)["json"] + check(len(occurrences(client4, member, target, text)) == 1, "uncertain retry duplicated message") + check(result["text"] == text, "uncertain retry returned wrong message") + case("idempotency_after_disconnected_client", uncertain_replay) + + def timed_out_replay(): + client5, owner, member, outsider, target, _ = bootstrap(config) + key, text = unique("timeout-key"), unique("timeout") + upstream = urlsplit(client5.base) + forwarded = threading.Event() + release = threading.Event() + upstream_result = {} + + class DelayedResponse(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_POST(self): + connection = None + try: + length = int(self.headers["Content-Length"]) + body = self.rfile.read(length) + connection_type = (http.client.HTTPSConnection if upstream.scheme == "https" + else http.client.HTTPConnection) + options = {"timeout": 5} + if upstream.scheme == "https": + options["context"] = ssl.create_default_context(cafile=config.get("ca_file")) + connection = connection_type(upstream.hostname, upstream.port or + (443 if upstream.scheme == "https" else 80), **options) + headers = {k: v for k, v in self.headers.items() + if k.lower() not in ("host", "connection", "content-length")} + headers["Content-Length"] = str(len(body)) + connection.request("POST", self.path, body=body, headers=headers) + response = connection.getresponse() + payload = response.read() + upstream_result["status"] = response.status + upstream_result["body"] = payload + except Exception as exc: + upstream_result["error"] = type(exc).__name__ + finally: + forwarded.set() + release.wait(5) + if connection: + connection.close() + try: + self.send_response(upstream_result.get("status", 502)) + self.end_headers() + self.wfile.write(upstream_result.get("body", b"")) + except OSError: + pass + + proxy = ThreadingHTTPServer(("127.0.0.1", 0), DelayedResponse) + proxy.daemon_threads = True + server_thread = threading.Thread(target=proxy.serve_forever, daemon=True) + server_thread.start() + try: + short = Client("http://127.0.0.1:%d" % proxy.server_port, timeout=0.01) + try: + post(short, owner, target, {"text": text}, key) + except Failure as exc: + check("connection/TLS failure" in str(exc), "delayed request failed for a reason other than timeout") + else: + raise Failure("delayed response did not produce a client timeout") + check(forwarded.wait(5), "test proxy did not forward the timed-out POST") + check(upstream_result.get("status") == 201, + "timed-out POST did not commit through public LB: " + + str(upstream_result.get("error", upstream_result.get("status")))) + finally: + release.set() + proxy.shutdown() + proxy.server_close() + server_thread.join(timeout=5) + committed = occurrences(client5, member, target, text) + check(len(committed) == 1, "timed-out first request was not committed before retry") + result = post(client5, owner, target, {"text": text}, key)["json"] + check(result == committed[0] and len(occurrences(client5, member, target, text)) == 1, + "retry after timeout did not resolve to exactly one message") + case("idempotency_after_client_timeout", timed_out_replay) + failed = [name for name in executed if evidence.summary[name]["status"] == "FAIL"] + pending = [name for name in executed if evidence.summary[name]["status"] == "PENDING_INFRA"] + check(not failed, "idempotency cases failed: " + ", ".join(failed)) + if pending: + raise Pending("idempotency cases awaiting infrastructure: " + ", ".join(pending)) + + +def observe_instances(client, actor, count=60): + ids = set() + for _ in range(count): + response = client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + ids.add(response["headers"].get("X-Instance-Id")) + check(len(ids) >= 2, "public LB exposed fewer than two X-Instance-Id values") + return sorted(ids) + + +def wait_instances(client, actor, timeout=10): + deadline = time.monotonic() + timeout + while True: + try: + return observe_instances(client, actor, count=24) + except Failure: + if time.monotonic() >= deadline: + raise + time.sleep(.25) + + +def load_window(config, evidence, client, actor, chat, seconds, label, expected_instance=None, + started=None, action_done=None, action_process=None): + start = time.monotonic() if started is None else started + successes, failures, reads, latencies, observed = 0, 0, 0, [], set() + after_action = set() + started_at_utc = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + error_events = [] + first_good, first_read = None, None + pending_key = uuid.uuid4().hex + pending_text = "lab4-" + uuid.uuid4().hex + attempted_texts, confirmed_texts = set(), set() + maximum = float(config.get("rolling_max_seconds", 120)) + while (time.monotonic() - start < seconds or + (action_process is not None and action_process.poll() is None)) and time.monotonic() - start < maximum: + settled = action_done is None or action_done.is_set() + attempted_texts.add(pending_text) + request_started_at_utc = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + tick = time.monotonic() + try: + response = post(client, actor, chat, {"text": pending_text}, pending_key) + successes += 1 + confirmed_texts.add(pending_text) + latencies.append(response["elapsed"]) + observed.add(response["headers"].get("X-Instance-Id")) + if settled: + after_action.add(response["headers"].get("X-Instance-Id")) + evidence.record("request", label=label, operation="POST", status=201, + seconds=round(time.monotonic() - start, 6), + elapsed_seconds=round(time.monotonic() - tick, 6), + instance=response["headers"].get("X-Instance-Id"), + message_id=response["json"]["id"], after_action=settled) + if settled and first_good is None: + first_good = time.monotonic() - start + pending_key, pending_text = uuid.uuid4().hex, "lab4-" + uuid.uuid4().hex + except Failure as exc: + failures += 1 + evidence.record("request", label=label, operation="POST", status="error", + seconds=round(time.monotonic() - start, 6), + elapsed_seconds=round(time.monotonic() - tick, 6), detail=str(exc)) + error_events.append({"seconds": round(time.monotonic() - start, 3), + "operation": "POST", "detail": str(exc)[:160], + "request_started_at_utc": request_started_at_utc, + "failed_at_utc": datetime.now(timezone.utc).isoformat(timespec="milliseconds")}) + request_started_at_utc = datetime.now(timezone.utc).isoformat(timespec="milliseconds") + settled = action_done is None or action_done.is_set() + tick = time.monotonic() + try: + response = client.call("GET", message_path(chat) + "?limit=1", actor=actor, schema="MessagePage") + reads += 1 + if settled and first_read is None: + first_read = time.monotonic() - start + observed.add(response["headers"].get("X-Instance-Id")) + if settled: + after_action.add(response["headers"].get("X-Instance-Id")) + evidence.record("request", label=label, operation="GET", status=200, + seconds=round(time.monotonic() - start, 6), + elapsed_seconds=round(time.monotonic() - tick, 6), + instance=response["headers"].get("X-Instance-Id"), after_action=settled) + except Failure as exc: + failures += 1 + evidence.record("request", label=label, operation="GET", status="error", + seconds=round(time.monotonic() - start, 6), + elapsed_seconds=round(time.monotonic() - tick, 6), detail=str(exc)) + error_events.append({"seconds": round(time.monotonic() - start, 3), + "operation": "GET", "detail": str(exc)[:160], + "request_started_at_utc": request_started_at_utc, + "failed_at_utc": datetime.now(timezone.utc).isoformat(timespec="milliseconds")}) + elapsed = time.monotonic() - start + summary = {"label": label, "duration_seconds": round(elapsed, 3), + "started_at_utc": started_at_utc, + "ended_at_utc": datetime.now(timezone.utc).isoformat(timespec="milliseconds"), + "successful_write_rps": round(successes / elapsed, 3), + "successful_writes": successes, "successful_reads": reads, + "successful_read_rps": round(reads / elapsed, 3), + "error_fraction": failures / (successes + reads + failures), + "errors": failures, "first_success_seconds": first_good, + "first_read_seconds": first_read, + "observed_instances": sorted(observed), "after_action_instances": sorted(after_action), + "latency_seconds": latencies, + "p50_seconds": percentile(latencies, .50), + "p95_seconds": percentile(latencies, .95), + "p99_seconds": percentile(latencies, .99), + "error_events": error_events} + evidence.record("load_window", **summary) + if action_process is not None: + check(action_process.poll() is not None, "rolling update exceeded the load time budget") + check(successes > 0 and reads > 0, label + ": no successful reads/writes") + if expected_instance: + check(expected_instance not in after_action, label + ": stopped/unready backend still served traffic") + summary["_attempted_texts"] = attempted_texts + summary["_confirmed_texts"] = confirmed_texts + return summary + + +def percentile(values, fraction): + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None + + +def verify_load_history(client, actor, chat, result): + messages = history(client, actor, chat) + texts = [m["text"] for m in messages if m["text"] in result["_attempted_texts"]] + check(len(texts) == len(set(texts)), "uncertain request created duplicate message") + check(result["_confirmed_texts"].issubset(set(texts)), "confirmed write disappeared under load") + + +def failover(config, evidence): + client, alice, bob, eve, chat, _ = bootstrap(config) + aliases = observe_instances(client, alice) + check(set(aliases) == {alias for alias, item in api_by_alias(config).items() if item["State"]["Running"]}, + "LB did not expose every running API replica with the same token") + evidence.record("lb_instances", aliases=aliases) + require_hooks(config, "stop_one", "start_one") + for target in aliases: + def one(): + before_docker = api_by_alias(config) + check(target in before_docker and len(before_docker) >= 2 and + all(item["State"]["Running"] for item in before_docker.values()), + "target alias does not map to one of two running API containers") + old = before_docker[target] + stable_key = uuid.uuid4().hex + stable_text = "lab4-before-" + uuid.uuid4().hex + before = post(client, alice, chat, {"text": stable_text}, stable_key)["json"] + stop_start = time.monotonic() + stopped_event = threading.Event() + def stop(): + hook(config, "stop_one", evidence, target=target, timeout=10) + stopped_event.set() + try: + with ThreadPoolExecutor(max_workers=1) as pool: + stopping = pool.submit(stop) + result = load_window(config, evidence, client_for(config, timeout=1), alice, + chat, float(config.get("failover_window_seconds", 15)), + "stop_" + target, expected_instance=target, + started=stop_start, action_done=stopped_event) + stopping.result() + stopped = api_by_alias(config) + check(target in stopped and stopped[target]["Id"] == old["Id"] and + not stopped[target]["State"]["Running"] and + all(item["State"]["Running"] for alias, item in stopped.items() if alias != target), + "stop hook did not stop exactly the selected API container") + write_recovery = result["first_success_seconds"] + read_recovery = result["first_read_seconds"] + check(write_recovery is not None and read_recovery is not None, + "no successful reads/writes after stop completed") + evidence.record("failover_recovery", target_instance=target, + write_seconds=round(write_recovery, 3), read_seconds=round(read_recovery, 3)) + check(write_recovery <= 10 and read_recovery <= 10, + "read/write did not recover within 10 seconds from stop start") + check(before in history(client, bob, chat), "confirmed message lost after stop") + check(post(client, alice, chat, {"text": stable_text}, stable_key)["json"] == before, + "replay after replica stop returned different Message") + verify_load_history(client, bob, chat, result) + finally: + hook(config, "start_one", evidence, target=target, timeout=60) + restarted = api_by_alias(config) + check(target in restarted and restarted[target]["Id"] == old["Id"] and + restarted[target]["State"]["Running"] and + restarted[target]["State"]["StartedAt"] != old["State"]["StartedAt"] and + volume_names(restarted[target]) == volume_names(old), + "selected API container did not restart with same ID/volumes") + evidence.record("failover_docker", target_instance=target, + container_id=old["Id"], + started_at_before=old["State"]["StartedAt"], + started_at_after=restarted[target]["State"]["StartedAt"], + volumes=volume_names(old)) + deadline = time.monotonic() + 30 + while True: + try: + observed = observe_instances(client, alice) + if len(observed) >= 2: + break + except Failure: + pass + check(time.monotonic() < deadline, "second replica did not return through LB") + time.sleep(.25) + evidence.case("failover_" + target, one) + failed = [alias for alias in aliases if evidence.summary["failover_" + alias]["status"] != "PASS"] + check(not failed, "replica failover failed: " + ", ".join(failed)) + + +def grade4(config, evidence, only=None): + def case(name, fn): + if only is None or name in ("rolling_update_under_load", "rolling_update_zero_http_errors_quality"): + evidence.case(name, fn) + def health(): + client = public_ready(config) + client.call("GET", "/health/live", schema="Health") + client.call("GET", "/health/ready", schema="Health") + evidence.record("health", live=200, ready=200) + case("health_live_ready", health) + + def topology(): + argv = config.get("compose_config_argv", ["docker", "compose", "config", "--format", "json"]) + try: + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, timeout=15, check=False) + except (OSError, subprocess.TimeoutExpired): + raise Pending("Docker Compose config unavailable") from None + if proc.returncode: + raise Pending("Docker Compose config unavailable; provide .env/secrets and compose_config_argv") + try: + services = json.loads(proc.stdout)["services"] + except (ValueError, KeyError): + raise Failure("Compose config did not return services JSON") from None + apis = config.get("api_services") or [name for name in services if name == "api" or name.startswith("api-")] + lbs = config.get("lb_services") or [name for name in services if name in ("lb", "proxy", "nginx", "haproxy", "traefik")] + issues = [] + if not lbs or not all(name in services for name in lbs): + issues.append("no configured LB service") + if not apis or not all(name in services for name in apis): + issues.append("no configured API service") + runtime = {} + project = None + for name, service in services.items(): + deploy = service.get("deploy", {}) + limits = deploy.get("resources", {}).get("limits", {}) + if not ((limits.get("cpus") or service.get("cpus")) and + (limits.get("memory") or service.get("mem_limit"))): + issues.append(name + " lacks CPU/RAM limits") + ids = compose_ids(config, name, all_containers=True) + if not ids: + issues.append(name + " has no Docker container") + runtime[name] = [] + for identifier in ids: + raw = inspect_container(identifier) + labels = raw.get("Config", {}).get("Labels", {}) + observed_project = labels.get("com.docker.compose.project") + if project is None: + project = observed_project + if not observed_project or observed_project != project or labels.get("com.docker.compose.service") != name: + issues.append(name + " container has wrong Compose ownership") + host = raw.get("HostConfig", {}) + cpu = host.get("NanoCpus", 0) + memory = host.get("Memory", 0) + if not (isinstance(cpu, int) and cpu > 0 and isinstance(memory, int) and memory > 0): + issues.append(name + " lacks enforced Docker CPU/RAM limits") + runtime[name].append({"id": raw["Id"], "running": raw["State"]["Running"], + "exit_code": raw["State"].get("ExitCode"), + "nano_cpus": cpu, "memory_bytes": memory, + "healthcheck": bool(raw.get("Config", {}).get("Healthcheck")), + "port_bindings": host.get("PortBindings") or {}}) + for name in apis: + if name not in services: + continue + service = services[name] + if service.get("ports"): + issues.append(name + " publishes an API port around the LB") + if service.get("container_name"): + issues.append(name + " has a scaling-blocking container_name") + if not service.get("healthcheck") and not all(x["healthcheck"] for x in runtime[name]): + issues.append(name + " has no healthcheck") + if not service.get("stop_grace_period"): + issues.append(name + " has no stop_grace_period") + replicas = sum(int(services[name].get("deploy", {}).get("replicas", + services[name].get("scale", 1))) for name in apis if name in services) + if replicas < 2: + issues.append("fewer than two API replicas declared") + if sum(x["running"] for name in apis for x in runtime.get(name, [])) < 2: + issues.append("fewer than two running API containers") + for name in apis: + if any(x["port_bindings"] for x in runtime.get(name, [])): + issues.append(name + " has a direct Docker host port binding") + migrate = services.get("migrate") + if not migrate: + issues.append("migration service missing") + evidence.record("compose_topology", api_services=apis, declared_replicas=replicas, + lb_services=lbs, migration_service="migrate" if migrate else None, + project=project, runtime=runtime, issues=issues) + check(not issues, "; ".join(issues)) + case("compose_lb_limits_ports_migration_layout", topology) + + def readiness_exclusion(): + client, alice, bob, eve, chat, _ = bootstrap(config) + aliases = observe_instances(client, alice) + target = aliases[0] + unready_start = time.monotonic() + require_hooks(config, "make_unready", "restore_ready", "probe_health") + try: + hook(config, "make_unready", evidence, target=target) + direct = probe(config, evidence, "probe_health", target) + check(direct.get("live_status") == 200 and direct.get("ready_status") == 503, + "unready replica must remain live and return ready=503") + deadline = unready_start + 10 + while True: + try: + observed = [client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + ["headers"]["X-Instance-Id"] for _ in range(12)] + if target not in observed: + break + except Failure: + pass + check(time.monotonic() < deadline, "LB did not exclude unready replica within 10 seconds") + time.sleep(.2) + evidence.record("readiness_exclusion", target_instance=target, + elapsed_seconds=round(time.monotonic() - unready_start, 3)) + result = load_window(config, evidence, client, alice, chat, 3, "unready", expected_instance=target) + check(result["errors"] == 0, "LB still sent requests to unready backend") + finally: + hook(config, "restore_ready", evidence, target=target) + wait_instances(client, alice, timeout=10) + case("lb_excludes_unready_replica", readiness_exclusion) + + def lb_timing(): + argv = config.get("hooks", {}).get("lb_timing_evidence") + if not isinstance(argv, list) or not argv: + raise Pending("configure hooks.lb_timing_evidence for LB health interval/timeout") + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, timeout=10, check=False) + check(proc.returncode == 0, "LB timing evidence hook failed") + try: + value = json.loads(proc.stdout) + except ValueError: + raise Failure("LB timing evidence hook must return JSON") from None + interval = value.get("health_interval_seconds") + timeout = value.get("health_timeout_seconds") + detection = value.get("configured_max_detection_seconds") + check(all(isinstance(x, (int, float)) and not isinstance(x, bool) and x > 0 + for x in (interval, timeout, detection)), "LB timing values must be positive numbers") + check(detection <= 10, "configured LB detection exceeds 10 seconds") + evidence.record("lb_timing", health_interval_seconds=interval, + health_timeout_seconds=timeout, + configured_max_detection_seconds=detection) + case("lb_health_timing", lb_timing) + + def sigterm(): + client, alice, bob, eve, chat, _ = bootstrap(config) + target = wait_instances(client, alice)[0] + require_hooks(config, "sigterm_one", "start_one", "probe_stopped") + try: + hook(config, "sigterm_one", evidence, target=target, timeout=25) + direct = probe(config, evidence, "probe_stopped", target) + check(direct.get("accepting") is False, "terminated replica still accepts requests") + check(direct.get("signal") == "SIGTERM" and direct.get("forced_kill") is False and + direct.get("exit_code") == 0 and direct.get("connection_probe_performed") is True, + "probe must prove SIGTERM exit without SIGKILL and attempt a new connection") + result = load_window(config, evidence, client, alice, chat, 10, "sigterm", expected_instance=target) + verify_load_history(client, bob, chat, result) + finally: + hook(config, "start_one", evidence, target=target, timeout=60) + case("sigterm_drains_and_stops_accepting", sigterm) + + rolling_result = {} + def rolling(): + client, alice, bob, eve, chat, _ = bootstrap(config) + before = wait_instances(client, alice) + old_containers = [inspect_container(identifier) for identifier in api_ids(config)] + old_ids = {item["Id"] for item in old_containers} + check(len(old_ids) == 2 and all(item["State"]["Running"] for item in old_containers), + "rolling update requires two running API containers") + argv = config.get("hooks", {}).get("rolling_update") + if not isinstance(argv, list) or not argv: + raise Pending("configure hooks.rolling_update for sequential replacement under load") + telemetry = bool(config.get("rolling_telemetry")) + captures = [] + since = str(int(time.time())) + telemetry_dir = evidence.directory / "rolling-telemetry" + if telemetry: + telemetry_dir.mkdir(exist_ok=True) + project = old_containers[0]["Config"]["Labels"]["com.docker.compose.project"] + lb_ids = compose_ids(config, "lb") + check(len(lb_ids) == 1, "rolling telemetry needs one LB container") + def capture(name, command): + output = (telemetry_dir / (name + ".log")).open("wb") + errors = (telemetry_dir / (name + ".stderr.txt")).open("wb") + watcher = subprocess.Popen(command, cwd=ROOT, stdout=output, stderr=errors) + captures.append((watcher, output, errors)) + capture("docker-events", ["docker", "events", "--filter", "type=container", + "--filter", "label=com.docker.compose.project=" + project, + "--format", "{{json .}}"]) + capture("haproxy", ["docker", "logs", "--follow", "--timestamps", "--since", since, lb_ids[0]]) + for identifier in sorted(old_ids): + capture("api-old-" + identifier[:12], + ["docker", "logs", "--follow", "--timestamps", "--since", since, identifier]) + time.sleep(.2) + rolling_env = dict(os.environ) + if telemetry: + rolling_env["HL_ROLLING_TRACE_ACTIVE"] = "1" + process = subprocess.Popen(argv, cwd=ROOT, env=rolling_env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, start_new_session=True) + try: + result = load_window(config, evidence, client_for(config, timeout=1), alice, chat, + float(config.get("rolling_window_seconds", 20)), "rolling", + action_process=process) + output, errors = process.communicate(timeout=5) + if telemetry: + (telemetry_dir / "rolling-script.stdout.txt").write_bytes(output) + (telemetry_dir / "rolling-script.stderr.txt").write_bytes(errors) + check(process.returncode == 0, "rolling update hook failed") + try: + lifecycle = json.loads(output) + except ValueError: + raise Failure("rolling hook must return lifecycle JSON") from None + check(lifecycle.get("replaced_processes", 0) >= 2 and + lifecycle.get("min_ready_during_update", 0) >= 1, + "rolling hook did not prove sequential replacement and a ready backend") + after = observe_instances(client, alice) + new_containers = [inspect_container(identifier) for identifier in api_ids(config)] + new_ids = {item["Id"] for item in new_containers} + check(len(new_ids) == 2 and not old_ids.intersection(new_ids) and + all(item["State"]["Running"] for item in new_containers), + "rolling update did not replace both real API containers") + validate_rolling_steps(lifecycle, old_ids, new_ids) + verify_load_history(client, bob, chat, result) + evidence.record("rolling_update", before=before, after=after, + old_container_ids=sorted(old_ids), new_container_ids=sorted(new_ids), + replaced_processes=lifecycle["replaced_processes"], + min_ready_during_update=lifecycle["min_ready_during_update"], + steps=lifecycle["steps"], + errors=result["errors"], successful_write_rps=result["successful_write_rps"]) + rolling_result.update(result) + finally: + if process.poll() is None: + process.terminate() + process.wait(timeout=5) + if telemetry: + for watcher, output_stream, error_stream in captures: + if watcher.poll() is None: + watcher.terminate() + watcher.wait(timeout=5) + output_stream.close() + error_stream.close() + try: + for identifier in compose_ids(config, "api"): + if identifier not in old_ids: + proc = subprocess.run(["docker", "logs", "--timestamps", "--since", since, + identifier], cwd=ROOT, capture_output=True, timeout=10) + (telemetry_dir / ("api-new-" + identifier[:12] + ".log")).write_bytes( + proc.stdout + proc.stderr) + except (OSError, subprocess.SubprocessError, Pending): + evidence.record("rolling_telemetry_incomplete", reason="new API logs unavailable") + case("rolling_update_under_load", rolling) + def rolling_quality(): + if not rolling_result: + raise Pending("rolling update did not complete; inspect rolling_update_under_load") + check(rolling_result["errors"] == 0, "rolling update had %d HTTP errors" % rolling_result["errors"]) + case("rolling_update_zero_http_errors_quality", rolling_quality) + + def migration_once(): + # A one-time historical fact requires lifecycle evidence, not just a current + # Compose snapshot. An adapter must count actual migration starts for this run. + argv = config.get("hooks", {}).get("migration_evidence") + if not isinstance(argv, list) or not argv: + raise Pending("configure hooks.migration_evidence to count actual migration starts") + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, timeout=20, check=False) + check(proc.returncode == 0, "migration evidence hook failed") + try: + value = json.loads(proc.stdout) + except ValueError: + raise Failure("migration evidence hook must return JSON") from None + validate_migration_events(value) + evidence.record("migration_once", **value) + case("migrations_started_once", migration_once) + + +def public_checks(config, evidence): + def run_make(target, action=None, label=None): + argv = ["make", target, "PYTHON=" + sys.executable, "BASE_URL=" + config["base_url"]] + if config.get("ca_file"): + argv.append("CA_FILE=" + config["ca_file"]) + if action: + script = config.get("hooks", {}).get(action) + if not isinstance(script, list) or len(script) != 1: + raise Pending("public %s requires hooks.%s with one executable script path" % (target, action)) + path = Path(script[0]) + if not path.is_absolute(): + path = ROOT / path + if not path.is_file() or not os.access(path, os.X_OK): + raise Pending("public %s action script is missing or not executable" % target) + argv.append("ACTION_SCRIPT=" + str(path)) + started = time.monotonic() + try: + proc = subprocess.run(argv, cwd=ROOT, capture_output=True, timeout=180, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure("%s failed to launch or timed out (%s)" % (target, type(exc).__name__)) from None + stem = label or target + (evidence.directory / (stem + ".stdout.txt")).write_bytes(proc.stdout) + (evidence.directory / (stem + ".stderr.txt")).write_bytes(proc.stderr) + evidence.record("public_make", target=target, label=stem, exit_code=proc.returncode, + elapsed_seconds=round(time.monotonic() - started, 3)) + if proc.returncode: + public_ready(config) # An unreachable LB is PENDING_INFRA, never application PASS. + check(proc.returncode == 0, target + " failed; inspect its saved stdout/stderr") + evidence.case("public_make_test", lambda: run_make("test")) + evidence.case("public_make_test_persistence", lambda: run_make("test-persistence", "restart")) + def both_failovers(): + if not config.get("hooks", {}).get("recover_all"): + raise Pending("configure hooks.recover_all before public failover") + stopped_aliases = set() + for attempt in range(1, 5): + try: + before = api_by_alias(config) + except Pending: + # Still attempt the explicitly requested public command and save + # its original result even when Docker cannot supply proof. + run_make("test-failover", "stop_one", label="test-failover-%d" % attempt) + raise + check(len(before) == 2 and all(item["State"]["Running"] for item in before.values()), + "public failover requires two running API containers") + try: + run_make("test-failover", "stop_one", label="test-failover-%d" % attempt) + after = api_by_alias(config) + stopped = [alias for alias, item in after.items() if not item["State"]["Running"]] + check(len(stopped) == 1 and stopped[0] in before and + after[stopped[0]]["Id"] == before[stopped[0]]["Id"], + "public failover did not leave exactly its selected replica stopped") + stopped_aliases.add(stopped[0]) + evidence.record("public_failover_target", attempt=attempt, + target_instance=stopped[0], container_id=after[stopped[0]]["Id"]) + finally: + hook(config, "recover_all", evidence, timeout=60) + if len(stopped_aliases) == 2: + break + check(len(stopped_aliases) == 2, + "public make test-failover did not exercise both API replicas in four attempts") + evidence.case("public_make_test_failover_both_replicas", both_failovers) + + +def validate_rolling_steps(value, old_ids, new_ids): + steps = value.get("steps", []) + check(isinstance(steps, list) and len(steps) == len(old_ids), "rolling hook must provide each replacement step") + check({s.get("old_id") for s in steps} == old_ids and {s.get("new_id") for s in steps} == new_ids, + "rolling steps do not match real replaced containers") + last_end = 0 + for step in steps: + begin, end = step.get("begin_ns"), step.get("end_ns") + check(isinstance(begin, int) and isinstance(end, int) and last_end <= begin < end, + "rolling replacements overlap or have invalid timestamps") + survivors = step.get("ready_survivor_ids", []) + check(survivors and all(identifier in old_ids | new_ids and identifier != step["old_id"] + for identifier in survivors), "rolling step had no other ready backend") + last_end = end + + +def validate_migration_events(value): + check(value.get("capture_started_before_stack") is True, + "migration evidence must capture lifecycle from before stack startup") + events = value.get("events", []) + check(isinstance(events, list) and events, "raw migration/API lifecycle events are required") + starts = [e for e in events if e.get("service") == "migrate" and e.get("action") == "start"] + exits = [e for e in events if e.get("service") == "migrate" and e.get("action") == "die"] + apis = [e for e in events if e.get("service") == "api" and e.get("action") == "start"] + check(len(starts) == len(exits) == 1 and len({e.get("container_id") for e in apis}) >= 2, + "expected exactly one migration start/exit and at least two real API starts") + check(all(isinstance(e.get("time_ns"), int) and e["time_ns"] > 0 and e.get("container_id") + for e in starts + exits + apis), "lifecycle events require real IDs and nanosecond timestamps") + check(starts[0]["container_id"] == exits[0]["container_id"] and + starts[0]["time_ns"] < exits[0]["time_ns"] and exits[0].get("exit_code") == 0, + "migrator did not complete successfully") + check(all(e["time_ns"] >= exits[0]["time_ns"] for e in apis), + "API started before migration completed") + + +def attachments(config, evidence): + """Lab4-specific contract checks; reuse the public client and fixture.""" + content = (ROOT / "tests/fixtures/sample.png").read_bytes() + + def upload(client, actor, chat, data=content, mime="image/png", filename="sample.png", expected=201): + response = client.call("POST", "/api/v1/chats/" + chat["id"] + + "/attachments?filename=" + quote(filename, safe=""), expected, + actor=actor, raw=data, headers={"Content-Type": mime}, + schema="Attachment" if expected == 201 else None) + if expected == 201: + att = response["json"] + check(att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, + "upload must synchronously return ready metadata without variants/error") + check(att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"] and + att["sha256"] == hashlib.sha256(data).hexdigest() and att["size_bytes"] == len(data), + "attachment ownership or original checksum/size mismatch") + else: + expected_code = {400: "invalid_request", 404: "not_found", 413: "payload_too_large", + 415: "unsupported_media_type"}[expected] + check(response["json"]["error"]["code"] == expected_code, "incorrect upload error code") + return response + + def negative_uploads(): + c, alice, bob, eve, chat, other = bootstrap(config) + # The oversized fixture is generated in memory, never committed as a huge file. + upload(c, alice, chat, data=content + b"\0" * (10 * 1024 * 1024 + 1 - len(content)), expected=413) + upload(c, alice, chat, data=b"\x89PNG\r\n\x1a\ntruncated", expected=415) + upload(c, alice, chat, data=b"not an image", expected=415) + upload(c, alice, chat, mime="image/jpeg", expected=415) + upload(c, alice, chat, mime="application/octet-stream", expected=415) + for name in ("../image.png", "a\\b.png", "a\x00b.png", "a\x1fb.png", "", "x" * 256): + upload(c, alice, chat, filename=name, expected=400) + evidence.case("attachment_negative_uploads", negative_uploads) + + def acl_and_message(): + c, alice, bob, eve, chat, other = bootstrap(config) + att = upload(c, alice, chat)["json"] + path = "/api/v1/attachments/" + att["id"] + for suffix in ("", "/content"): + c.call("GET", path + suffix, 404, actor=eve) + upload(c, eve, chat, expected=404) + downloaded = c.call("GET", path + "/content", actor=bob) + check(downloaded["bytes"] == content and + downloaded["headers"].get_content_type() == "image/png", "original bytes/MIME changed") + # Reuse this end-to-end case rather than creating a second attachment suite. + attachments = [att] + [upload(c, alice, chat)["json"] for _ in range(9)] + ids = sorted((item["id"] for item in attachments), reverse=True) + for count in range(11): + body = {"text": "" if count else "Legacy text without attachments", "attachment_ids": ids[:count]} + if count == 1: + body["attachment_ids"] = [ids[0].upper()] + key = uuid.uuid4().hex + first = post(c, bob, chat, body, key)["json"] + if count == 1: + before = history(c, alice, chat) + replay = post(c, bob, chat, body, key)["json"] + check(replay == first, "identical uppercase-UUID JSON replay changed the 201/Message") + check(history(c, alice, chat) == before, "identical uppercase-UUID replay changed history") + check(first in history(c, alice, chat), "history lost message or attachment order for count=%d" % count) + for target, actor, attachment_id in ((other, alice, att["id"]), (chat, eve, att["id"]), + (chat, alice, str(uuid.uuid4()))): + before = history(c, alice, target) + post(c, actor, target, {"text": "negative reference", "attachment_ids": [ids[0], attachment_id]}, + uuid.uuid4().hex, 404) + check(history(c, alice, target) == before, "invalid attachment reference changed history") + key = uuid.uuid4().hex + original_body = {"text": "same", "attachment_ids": ids[:2]} + original = post(c, alice, chat, original_body, key)["json"] + before = history(c, bob, chat) + check(post(c, alice, chat, original_body, key)["json"] == original, "identical attachment replay changed Message") + check(history(c, bob, chat) == before, "identical attachment replay changed history") + other_att = upload(c, bob, chat)["json"] + for changed in (ids[:2][::-1], [ids[0], other_att["id"]], ids[:1], []): + post(c, alice, chat, {"text": "same", "attachment_ids": changed}, key, 409) + check(history(c, bob, chat) == before, "attachment list/order conflict changed history") + check(post(c, alice, chat, original_body, key)["json"] == original, + "attachment conflict changed original Message") + check(occurrences(c, bob, chat, "same") == [original], "attachment conflict wrote another message") + race_key = uuid.uuid4().hex + gate = threading.Barrier(int(config.get("concurrency", 12))) + def repeated(_): + gate.wait(timeout=10) + return post(client_for(config), bob, chat, {"text": "attachment race", "attachment_ids": ids[:3]}, race_key) + with ThreadPoolExecutor(max_workers=gate.parties) as pool: + responses = list(pool.map(repeated, range(gate.parties))) + raced = [response["json"] for response in responses] + aliases = sorted({response["headers"]["X-Instance-Id"] for response in responses}) + evidence.record("concurrent_attachment_idempotency", attempts=gate.parties, observed_instances=aliases) + check(len(aliases) >= 2, "concurrent attachment retries did not reach two LB backends") + check(all(message == raced[0] for message in raced), "concurrent attachment replay changed Message") + check(occurrences(c, alice, chat, "attachment race") == [raced[0]], "concurrent attachment replay duplicated history") + evidence.case("attachment_acl_bytes_and_idempotency", acl_and_message) + + # Faults and bucket policies are implementation-specific. A configured adapter + # executes the scenario and returns observations; a missing adapter is PENDING. + def operational(name): + require_hooks(config, name) + env = dict(os.environ, STUDENT_TEST_EVIDENCE_DIR=str(evidence.directory / name)) + proc = subprocess.run(config["hooks"][name], cwd=ROOT, env=env, capture_output=True, timeout=180, check=False) + if name == "attachment_bucket_security" and proc.returncode == 2: + raise Pending("S3 infrastructure unavailable; inspect attachment_bucket_security/summary.json") + check(proc.returncode == 0, name + " adapter failed") + try: + value = json.loads(proc.stdout) + except ValueError: + raise Failure(name + " adapter must return sanitized observation JSON") from None + validate_attachment_observations(name, value) + evidence.record(name, **value) + for name in ("attachment_compensation", "attachment_parallel_memory", "attachment_bucket_security", + "attachment_s3_failure", "attachment_pixels_limit"): + evidence.case(name, lambda name=name: operational(name)) + + +def validate_attachment_observations(name, value): + """Require measurements/observations, never accept a bare {pass: true}.""" + check(isinstance(value, dict), "adapter must return observation object") + if name == "attachment_compensation": + check(set(value.get("injected_faults", [])) >= {"client_disconnect", "after_s3_before_db"}, + "both interrupted upload and S3/DB boundary fault must be injected") + check(value.get("grace_seconds", 0) > 0 and value.get("untracked_objects_after_grace") == 0 and + value.get("fresh_objects_preserved") is True and value.get("referenced_objects_preserved") is True, + "cleanup did not prove grace period and preservation of fresh/referenced objects") + elif name == "attachment_parallel_memory": + check(value.get("concurrent_uploads", 0) >= 10 and value.get("successful_uploads", 0) >= 10, + "need at least 10 simultaneous successful uploads") + check(value.get("body_bytes", 0) > 0 and value.get("duration_seconds", 0) > 0, + "upload load profile missing") + samples = value.get("samples", []) + check(samples and all(s.get("time_seconds", -1) >= 0 and s.get("memory_bytes", 0) > 0 and + s.get("cpu_percent", -1) >= 0 for s in samples), "raw CPU/RAM samples missing") + check(0 < max(s["memory_bytes"] for s in samples) <= value.get("memory_limit_bytes", 0), + "memory was not bounded by the configured runtime limit") + check(value.get("oom_kills") == 0, "upload experiment had OOM kills") + elif name == "attachment_bucket_security": + check(value.get("bootstrap_runs", 0) >= 2 and value.get("bucket_survived_second_bootstrap") is True, + "bucket bootstrap not repeated with existing data") + check(value.get("anonymous_get_status") in (401, 403, 404) and + value.get("anonymous_list_status") in (401, 403, 404) and + value.get("api_credentials_are_admin") is False and + value.get("api_can_administer_other_buckets") is False and + value.get("console_reachable_from_shared_network") is False, + "bucket/API policy or console isolation failed") + elif name == "attachment_s3_failure": + check(value.get("fault") == "s3_unreachable" and value.get("upload_status") == 503 and + value.get("ready_status") == 503 and value.get("live_status") == 200 and + value.get("false_ready_attachments") == 0 and value.get("restored_original_checksum_equal") is True, + "S3 failure was not reported safely or original was lost") + elif name == "attachment_pixels_limit": + check(value.get("width", 0) * value.get("height", 0) > 40000000 and + 0 < value.get("encoded_bytes", 0) <= 10485760 and value.get("upload_status") == 413, + "need a valid image above pixel limit but below byte limit, rejected with 413") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, help="JSON config with PUBLIC LB address and optional hooks") + parser.add_argument("--output", required=True, help="directory for raw, token-free results") + parser.add_argument("--scenario", choices=["idempotency", "restart", "failover", "grade4", "rolling", "public", "attachments", "all"], default="all") + args = parser.parse_args() + evidence = Evidence(args.output) + try: + config = json.loads(Path(args.config).read_text(encoding="utf-8")) + check(isinstance(config, dict), "config must be a JSON object") + client_for(config) + check(2 <= int(config.get("concurrency", 12)) <= 64, "concurrency must be 2..64") + check(11 <= float(config.get("failover_window_seconds", 15)) <= 60, + "failover_window_seconds must be 11..60 to observe the 10 second budget") + check(1 <= float(config.get("rolling_window_seconds", 20)) <= 60 and + 60 <= float(config.get("rolling_max_seconds", 120)) <= 300, + "invalid rolling load time budget") + except (OSError, ValueError, Failure, Pending) as exc: + evidence.case("configuration", lambda: (_ for _ in ()).throw(Failure(str(exc)))) + return evidence.finish() + evidence.record("run_profile", scenario=args.scenario, base_url=config["base_url"], + concurrency=config.get("concurrency", 12), request_timeout_seconds=config.get("request_timeout_seconds", 3), + failover_window_seconds=config.get("failover_window_seconds", 15), + rolling_window_seconds=config.get("rolling_window_seconds", 20), + load_model="sequential closed-loop POST+GET; concurrency only for duplicate POST race") + evidence.case("public_lb_endpoint", lambda: public_endpoint(config, evidence)) + if args.scenario in ("idempotency", "restart", "all"): + only = "idempotency_survives_restart" if args.scenario == "restart" else None + evidence.case("idempotency_setup_and_suite", lambda: idempotency(config, evidence, only)) + if args.scenario in ("failover", "all"): + evidence.case("public_lb_failover", lambda: failover(config, evidence)) + if args.scenario in ("grade4", "all"): + grade4(config, evidence) + if args.scenario == "rolling": + grade4(config, evidence, only="rolling") + if args.scenario in ("attachments", "all"): + attachments(config, evidence) + if args.scenario in ("public", "all"): + public_checks(config, evidence) + return evidence.finish() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/lab4/s3_infrastructure.py b/student-tests/lab4/s3_infrastructure.py new file mode 100644 index 00000000..138f9644 --- /dev/null +++ b/student-tests/lab4/s3_infrastructure.py @@ -0,0 +1,365 @@ +#!/usr/bin/env python3 +"""Runtime S3 checks for an explicitly selected synthetic Compose project. + +Reuses lab4 Evidence/Client. Secrets stay in the existing one-shot client's +secret mounts; raw rc output (which may contain account IDs) is never saved. +""" +import argparse +import base64 +import contextlib +import hashlib +import io +import json +import os +from pathlib import Path +import re +import shlex +import subprocess +import sys +import time +import uuid + +import run as checks + +BUCKET = "messenger-attachments" +POLICY = "messenger-attachments-api-v1" + + +def validate_api_policy(user, envelope): + """Validate actual attached grants, including absence of inherited groups.""" + checks.check(user.get("status") == "enabled" and user.get("policies") == [POLICY] and + user.get("memberOf") == [], "API identity has unexpected policies/groups or is disabled") + document = envelope.get("policy", {}).get("policy", {}) + checks.check(isinstance(document, dict) and document.get("Version") == "2012-10-17", + "runtime policy document missing") + granted = set() + for statement in document.get("Statement", []): + checks.check(statement.get("Effect") == "Allow" and "NotAction" not in statement and + "NotResource" not in statement, "unexpected policy statement") + actions, resources = statement.get("Action"), statement.get("Resource") + actions = [actions] if isinstance(actions, str) else actions + resources = [resources] if isinstance(resources, str) else resources + checks.check(isinstance(actions, list) and isinstance(resources, list), "policy grant arrays missing") + granted.update((action, resource) for action in actions for resource in resources) + bucket = "arn:aws:s3:::" + BUCKET + expected = {("s3:ListBucket", bucket), ("s3:GetBucketLocation", bucket), + ("s3:PutObject", bucket + "/*"), ("s3:GetObject", bucket + "/*"), + ("s3:DeleteObject", bucket + "/*")} + checks.check(granted == expected, "API policy contains unexpected IAM/admin/bucket grants") + return [{"action": action, "resource": resource} for action, resource in sorted(granted)] + + +def denied(result): + """A network/CLI error must never pass as an authorization denial.""" + text = result.stdout + result.stderr + return result.returncode != 0 and bool(re.search(rb"access.?denied|\b403\b|\bforbidden\b", text, re.I)) + + +class Stack: + def __init__(self, project, evidence): + if project != "highload-lab4" and not re.fullmatch(r"lab4-(s3-test|final)-[a-z0-9-]+", project): + raise checks.Failure("use highload-lab4 or an isolated lab4-s3-test-* or lab4-final-* project") + self.project, self.evidence = project, evidence + self.base = ["docker", "compose", "--project-name", project] + self.nonce = uuid.uuid4().hex + self.prefix = "student-tests/s3-" + self.nonce + "/" + self.key = "roundtrip-" + self.nonce + ".bin" + self.control = "student-control-" + self.nonce + self.forbidden = "student-forbidden-" + self.nonce + self.payload = bytes(range(256)) * 257 + self.nonce.encode() + self.digest = hashlib.sha256(self.payload).hexdigest() + self.failures, self.bootstrap_runs = [], 0 + + def execute(self, argv, input=None, timeout=30): + try: + return subprocess.run(argv, input=input, cwd=checks.ROOT, capture_output=True, + timeout=timeout, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise checks.Failure("test command unavailable/timed out: " + type(exc).__name__) from None + + def docker_json(self, argv): + result = self.execute(argv) + if result.returncode: + raise checks.Pending("Docker inspection unavailable for selected project") + return json.loads(result.stdout) + + def containers(self, service): + result = self.execute(self.base + ["ps", "-aq", service]) + if result.returncode: + raise checks.Pending("Docker Compose inspection unavailable") + values = [] + for identifier in result.stdout.decode().split(): + value = self.docker_json(["docker", "inspect", identifier])[0] + labels = value["Config"].get("Labels", {}) + checks.check(labels.get("com.docker.compose.project") == self.project and + labels.get("com.docker.compose.service") == service, "container ownership mismatch") + values.append(value) + checks.check(values, "selected project has no " + service + " container") + return values + + def s3(self): + values = self.containers("s3") + checks.check(len(values) == 1, "expected one owned RustFS container") + return values[0] + + @staticmethod + def snapshot(value): + return {"id": value["Id"], "started_at": value["State"]["StartedAt"], + "image": value["Config"]["Image"], "running": value["State"]["Running"], + "volumes": checks.volume_names(value)} + + def client(self, command, root=False, payload=False): + # Only allow generated bucket/key paths and fixed argv. No credentials + # enter host Python variables, command arguments or evidence files. + setup = '''set -eu +rc=/usr/bin/rc +app_key=$(cat /run/secrets/s3_access_key_id) +app_secret=$(cat /run/secrets/s3_secret_access_key) +root_key=$(cat /run/secrets/rustfs_root_access_key) +root_secret=$(cat /run/secrets/rustfs_root_secret_key) +test "$app_key" != "$root_key" +"$rc" alias set app http://s3:9000 "$app_key" "$app_secret" --region us-east-1 --bucket-lookup path >/dev/null 2>&1 +''' + if root: + setup += '"$rc" alias set root http://s3:9000 "$root_key" "$root_secret" --region us-east-1 --bucket-lookup path >/dev/null 2>&1\n' + if payload: + setup += "printf '%s' " + shlex.quote(base64.b64encode(self.payload).decode()) + " | base64 -d > /tmp/source\n" + result = self.execute(self.base + ["run", "--rm", "--no-deps", "-T", "--entrypoint", "/bin/sh", "s3-init", "-s"], + input=(setup + command + "\n").encode(), timeout=40) + return result + + def rc(self, args, *, root=False, payload=False, expect_denied=False, label): + command = '"$rc" --json ' + shlex.join(args) + result = self.client(command, root=root, payload=payload) + refusal = denied(result) + self.evidence.record("s3_operation", operation=label, exit_code=result.returncode, + authorization_denied=refusal, + s3_error_code="AccessDenied" if refusal else None) + checks.check(refusal if expect_denied else result.returncode == 0, + label + (": expected S3 authorization denial" if expect_denied else ": S3 operation failed")) + return result + + def verify_download(self, label): + command = '"$rc" --json object copy ' + shlex.quote("app/" + BUCKET + "/" + self.prefix + self.key) + ''' /tmp/received >/dev/null 2>&1 +cmp /tmp/source /tmp/received +sha256sum /tmp/received +''' + result = self.client(command, payload=True) + digest = result.stdout.decode(errors="replace").split()[0] if result.stdout.strip() else "" + checks.check(result.returncode == 0 and digest == self.digest, + label + ": downloaded bytes differ or GET failed") + self.evidence.record("s3_roundtrip", phase=label, byte_equal=True, size_bytes=len(self.payload), sha256=digest) + + def unsigned(self, path): + result = self.execute(self.base + ["exec", "-T", "s3", "curl", "-sS", "--max-time", "5", + "-o", "/dev/null", "-w", "%{http_code}", "http://127.0.0.1:9000/" + path]) + checks.check(result.returncode == 0 and re.fullmatch(rb"\d{3}", result.stdout), "unsigned HTTP probe failed") + return int(result.stdout) + + def wait_s3(self): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + value = self.s3() + if value["State"]["Running"] and value["State"].get("Health", {}).get("Status") == "healthy": + return value + time.sleep(.5) + raise checks.Failure("RustFS did not regain Docker health in 60 seconds") + + def bootstrap(self): + # Reuses the actual bootstrap mounted by the existing service. Does not + # recreate API/DB, alter Compose or delete/recreate the RustFS volume. + for _ in range(2): + before = self.snapshot(self.s3()) + result = self.execute(self.base + ["run", "--rm", "--no-deps", "-T", "s3-init"], timeout=45) + self.evidence.record("s3_bootstrap", attempt=self.bootstrap_runs + 1, exit_code=result.returncode) + checks.check(result.returncode == 0, "repeated real bootstrap failed") + self.bootstrap_runs += 1 + checks.check(self.snapshot(self.s3()) == before, "bootstrap replaced/restarted RustFS or changed its volume") + self.verify_download("bootstrap_" + str(self.bootstrap_runs)) + + def restart(self): + before = self.snapshot(self.s3()) + started = time.monotonic() + result = self.execute(self.base + ["restart", "--no-deps", "--timeout", "20", "s3"], timeout=35) + checks.check(result.returncode == 0, "RustFS restart command failed") + after = self.snapshot(self.wait_s3()) + checks.check(before["id"] == after["id"] and before["volumes"] == after["volumes"] and + before["volumes"] and before["started_at"] != after["started_at"], + "restart did not preserve the real container/volume or restart its process") + self.evidence.record("s3_restart", before=before, after=after, ready_seconds=round(time.monotonic() - started, 3)) + self.verify_download("after_restart") + + def permissions(self): + target = "app/" + BUCKET + "/" + self.prefix + listing = self.rc(["bucket", "list", target], label="api_list_allowed_bucket") + checks.check(self.key.encode() in listing.stdout, "API LIST did not include its uploaded object") + # A real second bucket and known root-written sentinel distinguish a + # policy denial from a nonexistent bucket/object and network failures. + self.rc(["bucket", "create", "root/" + self.control], root=True, label="prepare_control_bucket") + self.rc(["object", "copy", "/tmp/source", "root/" + self.control + "/sentinel.bin"], + root=True, payload=True, label="prepare_control_object") + for label, args in [ + ("api_other_bucket_put", ["object", "copy", "/tmp/source", "app/" + self.control + "/api-write.bin"]), + ("api_other_bucket_get", ["object", "copy", "app/" + self.control + "/sentinel.bin", "/tmp/received"]), + ("api_other_bucket_delete", ["object", "remove", "app/" + self.control + "/sentinel.bin"]), + ("api_other_bucket_list", ["bucket", "list", "app/" + self.control]), + ("api_create_bucket", ["bucket", "create", "app/" + self.forbidden]), + ("api_iam_user_list", ["admin", "user", "list", "app/"]), + ("api_iam_policy_list", ["admin", "policy", "ls", "app/"]), + ]: + try: + self.rc(args, payload=True, expect_denied=True, label=label) + except checks.Failure as exc: + # Probe every boundary even if one permission is too broad. + self.failures.append(str(exc)) + checks.check(not self.failures, "; ".join(self.failures)) + + def private_bucket(self): + self.anonymous_get = self.unsigned(BUCKET + "/" + self.prefix + self.key) + self.anonymous_list = self.unsigned(BUCKET + "?list-type=2") + self.evidence.record("s3_unsigned", object_get_status=self.anonymous_get, bucket_list_status=self.anonymous_list) + checks.check(self.anonymous_get == 403 and self.anonymous_list == 403, + "known existing object and bucket must return HTTP 403 without a signature") + + def effective_policy(self): + # The user-info response contains an access key. Keep it only in memory; + # save the validated grants, never the original response/account fields. + user = self.client('"$rc" --json admin user info root/ "$app_key"', root=True) + policy = self.client('"$rc" --json admin policy info root/ ' + POLICY, root=True) + checks.check(user.returncode == policy.returncode == 0, "cannot read actual API policy binding") + try: + grants = validate_api_policy(json.loads(user.stdout), json.loads(policy.stdout)) + except ValueError: + raise checks.Failure("runtime IAM evidence was not valid JSON") from None + self.evidence.record("s3_effective_policy", policy_name=POLICY, inherited_groups=0, grants=grants, + root_credentials_distinct=True, iam_admin_grants=0) + + def console(self): + raw = self.s3() + networks = list(raw.get("NetworkSettings", {}).get("Networks", {})) + checks.check(len(networks) == 1, "select a single shared Compose network for the console probe") + network = networks[0] + labels = self.docker_json(["docker", "network", "inspect", network])[0].get("Labels", {}) + checks.check(labels.get("com.docker.compose.project") == self.project, "network ownership mismatch") + base = ["docker", "run", "--rm", "--pull=never", "--network", network, "--read-only", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges:true", "--entrypoint", "curl", raw["Config"]["Image"], + "-sS", "--connect-timeout", "2", "--max-time", "3", "-o", "/dev/null", "-w", "%{http_code}"] + control = self.execute(base + ["http://s3:9000/health/ready"]) + checks.check(control.returncode == 0 and control.stdout == b"200", "shared-network control probe failed") + result = self.execute(base + ["http://s3:9001/"]) + self.evidence.record("s3_console", control_status=200, console_curl_exit=result.returncode, + console_http_status=result.stdout.decode(errors="replace")[:3]) + checks.check(result.returncode == 7 and result.stdout == b"000", "console accepts connections on shared network") + + def delete_probe(self): + path = "app/" + BUCKET + "/" + self.prefix + self.key + self.rc(["object", "remove", path], label="api_delete_allowed_object") + listing = self.rc(["bucket", "list", "app/" + BUCKET + "/" + self.prefix], label="api_list_after_delete") + checks.check(self.key.encode() not in listing.stdout, "API DELETE left its known object present") + + def cleanup(self): + # Fixed generated paths only. Never recursive-delete the application + # bucket, remove volumes, reset IAM or touch pre-existing objects. + commands = [ + ["object", "remove", "root/" + BUCKET + "/" + self.prefix + self.key], + ["object", "remove", "root/" + self.control + "/sentinel.bin"], + ["object", "remove", "root/" + self.control + "/api-write.bin"], + ["bucket", "remove", "root/" + self.control], + ["bucket", "remove", "root/" + self.forbidden], + ] + errors = [] + for args in commands: + try: + result = self.client('"$rc" --json ' + shlex.join(args), root=True) + # Missing generated objects/buckets are normal after an expected + # denial; any other error is retained while cleanup continues. + missing = bool(re.search(rb"nosuchkey|nosuchbucket|not.?found|\b404\b|does not exist", result.stdout + result.stderr, re.I)) + self.evidence.record("s3_cleanup", operation=" ".join(args[:2]), target=args[-1], + exit_code=result.returncode, already_absent=missing) + checks.check(result.returncode == 0 or missing, "could not clean generated S3 fixture") + except checks.Failure as exc: + errors.append(str(exc)) + checks.check(not errors, "; ".join(errors)) + + +def topology(stack, base_url, evidence): + apis, lbs = stack.containers("api"), stack.containers("lb") + checks.check(len(apis) >= 2 and len(lbs) == 1, "expected >=2 API containers behind one LB") + for raw in apis: + checks.check(raw["State"]["Running"] and raw["State"].get("Health", {}).get("Status") == "healthy", + "API is not healthy") + checks.check(not any((raw["HostConfig"].get("PortBindings") or {}).values()), "API has a direct host port") + config = {"base_url": base_url, "compose_base_argv": stack.base, "api_services": ["api"], "lb_services": ["lb"]} + # Bind the check to this explicit project; common inspection defaults would + # otherwise risk probing the concurrently running lab3 stack on port 8080. + checks.public_endpoint(config, evidence) + client = checks.client_for(config) + if not hasattr(stack, "actor"): + stack.actor = client.user("s3infra") + actor = stack.actor + aliases = checks.observe_instances(client, actor, count=32) + expected = {raw["Config"]["Hostname"] for raw in apis} + checks.check(set(aliases) == expected, "same token did not traverse every actual API replica") + evidence.record("s3_api_topology", api_ids=sorted(raw["Id"] for raw in apis), observed_instances=aliases, + lb_id=lbs[0]["Id"], lb_image=lbs[0]["Config"]["Image"], same_token=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project", required=True) + parser.add_argument("--base-url", default="http://127.0.0.1:8081") + parser.add_argument("--output") + parser.add_argument("--as-hook", action="store_true", help="emit only the existing attachment_bucket_security observation JSON") + args = parser.parse_args() + directory = args.output or os.environ.get("STUDENT_TEST_EVIDENCE_DIR") + if not directory: + if not args.as_hook: + parser.error("--output is required outside hook mode") + directory = checks.ROOT / "student-tests/results" / ("s3-" + uuid.uuid4().hex) + evidence = checks.Evidence(directory) + stack = Stack(args.project, evidence) + def execute(): + ready = stack.wait_s3() + evidence.record("s3_initial", **stack.snapshot(ready), project=args.project, + payload_size_bytes=len(stack.payload), payload_sha256=stack.digest, + fixture_prefix=stack.prefix, control_bucket=stack.control, create_probe_bucket=stack.forbidden) + checks.check(checks.volume_names(ready), "RustFS has no persistent named volume") + try: + stack.rc(["object", "copy", "/tmp/source", "app/" + BUCKET + "/" + stack.prefix + stack.key], + payload=True, label="api_put_allowed_object") + stack.verify_download("before_bootstrap") + evidence.case("s3_bootstrap_preserves_object_and_volume", stack.bootstrap) + evidence.case("s3_restart_preserves_original_bytes", stack.restart) + evidence.case("s3_unsigned_known_object_403", stack.private_bucket) + evidence.case("s3_api_bucket_scope_and_iam_denials", stack.permissions) + evidence.case("s3_api_effective_policy", stack.effective_policy) + evidence.case("s3_console_closed_on_shared_network", stack.console) + evidence.case("s3_api_delete_effect", stack.delete_probe) + evidence.case("s3_two_replicas_after_restart", lambda: topology(stack, args.base_url, evidence)) + finally: + evidence.case("s3_fixture_cleanup", stack.cleanup) + failed = [name for name, value in evidence.summary.items() if value["status"] == "FAIL"] + pending = [name for name, value in evidence.summary.items() if value["status"] == "PENDING_INFRA"] + checks.check(not failed, "S3 cases failed: " + ", ".join(failed)) + if pending: + raise checks.Pending("S3 cases pending: " + ", ".join(pending)) + capture = io.StringIO() + with contextlib.redirect_stdout(capture if args.as_hook else sys.stdout): + evidence.case("s3_two_replicas_before", lambda: topology(stack, args.base_url, evidence)) + evidence.case("s3_setup_and_scenarios", execute) + code = evidence.finish() + if code and all(value["status"] != "FAIL" for value in evidence.summary.values()): + code = 2 # PENDING_INFRA is distinct from a failed capability check. + if args.as_hook: + if code: + print("S3 checks incomplete/failed; inspect " + str(evidence.directory / "summary.json"), file=sys.stderr) + else: + print(json.dumps({"bootstrap_runs": stack.bootstrap_runs, "bucket_survived_second_bootstrap": True, + "anonymous_get_status": stack.anonymous_get, "anonymous_list_status": stack.anonymous_list, + "api_credentials_are_admin": False, "api_can_administer_other_buckets": False, + "console_reachable_from_shared_network": False})) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/lab4/test_harness.py b/student-tests/lab4/test_harness.py new file mode 100644 index 00000000..914d62c3 --- /dev/null +++ b/student-tests/lab4/test_harness.py @@ -0,0 +1,208 @@ +"""Regression checks for false-positive paths in the student test runner.""" +import contextlib +import io +import json +from pathlib import Path +import tempfile +import subprocess +import unittest +from unittest.mock import patch + +import run +import s3_infrastructure + + +class HarnessTests(unittest.TestCase): + def test_s3_cleanup_continues_after_error_and_never_removes_application_bucket(self): + stack = s3_infrastructure.Stack("highload-lab4", self.evidence()) + results = [subprocess.CompletedProcess([], 1, b"", b"AccessDenied")] + [ + subprocess.CompletedProcess([], 0, b"", b"") for _ in range(4)] + with patch.object(stack, "client", side_effect=results) as client: + with self.assertRaises(run.Failure): + stack.cleanup() + self.assertEqual(client.call_count, 5) + for call in client.call_args_list: + self.assertNotIn("bucket remove root/messenger-attachments", call.args[0]) + + def test_s3_policy_rejects_broad_or_inherited_grants(self): + bucket = "arn:aws:s3:::messenger-attachments" + user = {"status": "enabled", "policies": [s3_infrastructure.POLICY], "memberOf": []} + document = {"Version": "2012-10-17", "Statement": [ + {"Effect": "Allow", "Action": ["s3:ListBucket", "s3:GetBucketLocation"], "Resource": [bucket]}, + {"Effect": "Allow", "Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"], "Resource": [bucket + "/*"]}]} + envelope = {"policy": {"policy": document}} + s3_infrastructure.validate_api_policy(user, envelope) + for defect in ("admin", "other_bucket", "group"): + changed_user, changed = json.loads(json.dumps(user)), json.loads(json.dumps(envelope)) + if defect == "admin": + changed["policy"]["policy"]["Statement"][0]["Action"].append("admin:*") + elif defect == "other_bucket": + changed["policy"]["policy"]["Statement"][1]["Resource"] = ["arn:aws:s3:::*/*"] + else: + changed_user["memberOf"] = ["unexpected-group"] + with self.assertRaises(run.Failure): + s3_infrastructure.validate_api_policy(changed_user, changed) + + def test_missing_s3_infrastructure_emits_no_success_hook_json(self): + with tempfile.TemporaryDirectory() as directory: + args = ["s3_infrastructure.py", "--project", "highload-lab4", "--as-hook", "--output", directory] + output, errors = io.StringIO(), io.StringIO() + with patch("sys.argv", args), patch.object(s3_infrastructure, "Stack") as stack, \ + patch.object(s3_infrastructure, "topology", side_effect=run.Pending("no Docker")), \ + contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + stack.return_value.wait_s3.side_effect = run.Pending("no Docker") + code = s3_infrastructure.main() + self.assertEqual(code, 2) + self.assertEqual(output.getvalue(), "") + summary = json.loads((Path(directory) / "summary.json").read_text()) + self.assertTrue(all(case["status"] == "PENDING_INFRA" for case in summary.values())) + + def test_s3_network_failure_is_not_permission_denial(self): + for data in (b"connection refused", b"unrecognized subcommand", b"NoSuchBucket"): + self.assertFalse(s3_infrastructure.denied(subprocess.CompletedProcess([], 1, b"", data))) + self.assertFalse(s3_infrastructure.denied(subprocess.CompletedProcess([], 0, b"AccessDenied", b""))) + self.assertTrue(s3_infrastructure.denied(subprocess.CompletedProcess([], 1, b"", b'{"error":"AccessDenied (403)"}'))) + + def test_s3_rejects_unrelated_compose_project(self): + with self.assertRaises(run.Failure): + s3_infrastructure.Stack("highload-lab3", self.evidence()) + + def test_attachment_uuid_response_check_preserves_identity_and_order(self): + first = "aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee" + second = "bbbbbbbb-cccc-4ddd-8eee-ffffffffffff" + body = {"text": "", "attachment_ids": [first.upper(), second.upper()]} + actor, chat = {"id": "actor"}, {"id": "chat"} + + class Client: + def __init__(self, ids): + self.ids = ids + + def call(self, *args, **kwargs): + return {"json": {"chat_id": "chat", "sender_id": "actor", "text": "", "attachment_ids": self.ids}} + + run.post(Client([first, second]), actor, chat, body, "key") + for invalid in ([second, first], [first], [first, first]): + with self.assertRaises(run.Failure): + run.post(Client(invalid), actor, chat, body, "key") + + def evidence(self): + temp = tempfile.TemporaryDirectory() + self.addCleanup(temp.cleanup) + return run.Evidence(temp.name) + + def test_pending_and_failure_never_produce_success_exit(self): + for exception, expected in [(run.Pending("LB absent"), "PENDING_INFRA"), + (run.Failure("contract mismatch"), "FAIL")]: + ev = self.evidence() + def action(): + raise exception + with contextlib.redirect_stdout(io.StringIO()): + ev.case("example", action) + self.assertEqual(ev.finish(), 1) + self.assertEqual(ev.summary["example"]["status"], expected) + + def test_output_reuse_does_not_overwrite_raw_results(self): + ev = self.evidence() + ev.record("original") + with self.assertRaises(ValueError): + run.Evidence(ev.directory) + self.assertEqual(json.loads(ev.path.read_text())["event"], "original") + + def test_fault_needs_both_injection_and_restoration(self): + with self.assertRaises(run.Pending): + run.require_hooks({"hooks": {"make_unready": ["inject"]}}, "make_unready", "restore_ready") + + def test_credential_bearing_or_non_origin_urls_are_rejected(self): + for url in ["http://alice:secret@localhost:8080", "http://localhost:8080/path", + "http://localhost:8080?token=secret", "http://localhost:8080#fragment"]: + with self.assertRaises(run.Failure): + run.client_for({"base_url": url}) + + def snapshots(self): + before, after = {}, {} + for name in ["api", "postgres", "redis", "s3", "lb", "migrate"]: + value = {"id": name + "-id", "started_at": "before", "volumes": [name + "-data"], "running": True} + before[name] = [value] + after[name] = [{**value, "started_at": "before" if name == "migrate" else "after"}] + return before, after + + def test_restart_rejects_noop_store_and_volume_loss(self): + for defect in ["no_restart", "volume_loss", "migrate_reran"]: + before, after = self.snapshots() + if defect == "no_restart": + after["s3"][0]["started_at"] = "before" + elif defect == "volume_loss": + after["postgres"][0]["volumes"] = [] + else: + after["migrate"][0]["started_at"] = "after" + with self.assertRaises(run.Failure): + run.verify_restart(before, after, self.evidence()) + + def test_restart_accepts_observed_process_transition(self): + run.verify_restart(*self.snapshots(), self.evidence()) + + def events(self): + return {"capture_started_before_stack": True, "events": [ + {"service": "migrate", "action": "start", "container_id": "m", "time_ns": 1}, + {"service": "migrate", "action": "die", "container_id": "m", "time_ns": 2, "exit_code": 0}, + {"service": "api", "action": "start", "container_id": "a", "time_ns": 3}, + {"service": "api", "action": "start", "container_id": "b", "time_ns": 4}]} + + def test_migration_summary_without_raw_events_is_rejected(self): + with self.assertRaises(run.Failure): + run.validate_migration_events({"migration_starts": 1, "migration_exit_code": 0}) + + def test_duplicate_migration_or_early_api_is_rejected(self): + for defect in ["duplicate", "early_api", "failed_exit", "capture_too_late"]: + value = self.events() + if defect == "duplicate": + value["events"].append(dict(value["events"][0])) + elif defect == "early_api": + value["events"][2]["time_ns"] = 1 + elif defect == "failed_exit": + value["events"][1]["exit_code"] = 1 + else: + value["capture_started_before_stack"] = False + with self.assertRaises(run.Failure): + run.validate_migration_events(value) + + def test_one_successful_migration_before_two_apis_is_accepted(self): + run.validate_migration_events(self.events()) + + def test_rolling_summary_needs_nonoverlapping_real_container_steps(self): + with self.assertRaises(run.Failure): + run.validate_rolling_steps({"replaced_processes": 2, "min_ready_during_update": 1}, {"a", "b"}, {"c", "d"}) + value = {"steps": [ + {"old_id": "a", "new_id": "c", "begin_ns": 1, "end_ns": 3, "ready_survivor_ids": ["b"]}, + {"old_id": "b", "new_id": "d", "begin_ns": 4, "end_ns": 6, "ready_survivor_ids": ["c"]}]} + run.validate_rolling_steps(value, {"a", "b"}, {"c", "d"}) + value["steps"][1]["begin_ns"] = 2 + with self.assertRaises(run.Failure): + run.validate_rolling_steps(value, {"a", "b"}, {"c", "d"}) + + def test_adapters_cannot_return_only_pass_true(self): + for name in ["attachment_compensation", "attachment_parallel_memory", "attachment_bucket_security", + "attachment_s3_failure", "attachment_pixels_limit"]: + with self.assertRaises(run.Failure): + run.validate_attachment_observations(name, {"pass": True}) + + def test_load_history_rejects_duplicates_and_lost_confirmations(self): + result = {"_attempted_texts": {"a", "b"}, "_confirmed_texts": {"a"}} + for messages in [[{"text": "a"}, {"text": "a"}], [{"text": "b"}]]: + with patch.object(run, "history", return_value=messages), self.assertRaises(run.Failure): + run.verify_load_history(None, None, None, result) + + def test_public_lb_check_rejects_direct_api_port_binding(self): + def identifiers(_config, service, all_containers=False): + return [service] + def inspect(identifier): + if identifier == "lb": + return {"NetworkSettings": {"Ports": {"8080/tcp": [{"HostPort": "8080", "HostIp": "127.0.0.1"}]}}} + return {"HostConfig": {"PortBindings": {"8080/tcp": [{"HostPort": "8081"}]}}} + with patch.object(run, "compose_ids", identifiers), patch.object(run, "inspect_container", inspect): + with self.assertRaises(run.Failure): + run.public_endpoint({"base_url": "http://localhost:8080"}, self.evidence()) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/lab4/upload_memory.py b/student-tests/lab4/upload_memory.py new file mode 100644 index 00000000..631d30af --- /dev/null +++ b/student-tests/lab4/upload_memory.py @@ -0,0 +1,118 @@ +import argparse +import collections +import concurrent.futures +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote + +root = Path.cwd() +sys.path.insert(0, str(root / "tests")) +from client import Client, check + +parser = argparse.ArgumentParser(description="Measure memory on two disposable API replicas") +parser.add_argument("--base-url", required=True) +parser.add_argument("--project", required=True) +parser.add_argument("--fixtures", type=Path, required=True, help="Directory containing generated load.png and load.jpg") +parser.add_argument("--output", type=Path, required=True) +args = parser.parse_args() +out = args.output +out.mkdir(parents=True, exist_ok=True) +base, project, fixture_dir = args.base_url, args.project, args.fixtures + +def docker(*args): + return subprocess.check_output(["docker", *args], text=True, timeout=20).strip() + +def api_ids(): + return docker("ps", "-q", "--filter", "label=com.docker.compose.project=" + project, + "--filter", "label=com.docker.compose.service=api").splitlines() + +def memory(cid): + values = docker("exec", cid, "sh", "-c", "cd /sys/fs/cgroup && cat memory.current memory.peak memory.max memory.events").splitlines() + return dict(zip(("current", "peak", "limit"), map(int, values[:3]))) | { + "events": {k: int(v) for k, v in (line.split() for line in values[3:])}, + "state": json.loads(docker("inspect", "--format", "{{json .State}}", cid)), + "restarts": int(docker("inspect", "--format", "{{.RestartCount}}", cid)), + } + +def profile(name, files): + c = Client(base) + actor = c.user("load") + chat = c.call("POST", "/api/v1/chats", 201, actor=actor, body={ + "title": "Load " + name, "member_ids": [actor["id"]]}, schema="Chat")["json"] + ids = api_ids() + check(len(ids) == 2, "expected both API replicas") + before = {cid: memory(cid) for cid in ids} + workers = len(files) + gate = threading.Barrier(workers + 1) + samplers = [] + stop = threading.Event() + sampled = {cid: [] for cid in ids} + def sample(cid): + proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c", + "for i in $(seq 1 240); do read -r m = 10, "normal profile did not complete ten uploads") + +normal = (root / "tests/fixtures/sample.png").read_bytes() +png = (fixture_dir / "load.png").read_bytes() +jpeg = (fixture_dir / "load.jpg").read_bytes() +profile("normal", [("sample.png", "image/png", normal)] * 12) +profile("heavy", [("load.png", "image/png", png), ("load.jpg", "image/jpeg", jpeg)] * 6) diff --git a/student-tests/verify.py b/student-tests/verify.py new file mode 100644 index 00000000..dc48c1e6 --- /dev/null +++ b/student-tests/verify.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""One entry point: existing Go checks, runner regression checks, public and student scenarios.""" +import argparse +from collections import Counter +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--config", default="student-tests/lab4/config.example.json") + p.add_argument("--project", required=True, help="explicit synthetic Compose project to test") + p.add_argument("--output", required=True, help="new, empty directory for original results") + args = p.parse_args() + output = Path(args.output).resolve() + output.mkdir(parents=True, exist_ok=True) + if any(output.iterdir()): + p.error("output must be empty; never overwrite a previous experiment") + env = dict(os.environ, COMPOSE_PROJECT_NAME=args.project) + env.setdefault("GOCACHE", "/private/tmp/highload-lab4-go-cache" if sys.platform == "darwin" + else "/tmp/highload-lab4-go-cache") + summary = {"time_utc": datetime.now(timezone.utc).isoformat(), "project": args.project, + "scope": "lab4 student checks; no automatic grade", "checks": {}} + commands = [ + ("contract_structure", ["make", "check", "PYTHON=" + sys.executable], 30), + ("go_tests", ["go", "test", "-count=1", "-json", "./..."], 180), + ("runner_regressions", [sys.executable, "-m", "unittest", "discover", "-s", "student-tests/lab4", "-p", "test_*.py", "-v"], 30), + ("lab4_scenarios", [sys.executable, "student-tests/lab4/run.py", "--config", args.config, + "--output", str(output / "lab4"), "--scenario", "all"], 1800)] + for name, argv, timeout in commands: + print("Running " + name, flush=True) + try: + with (output / (name + ".stdout.txt")).open("wb") as stdout, (output / (name + ".stderr.txt")).open("wb") as stderr: + proc = subprocess.run(argv, cwd=ROOT, env=env, stdout=stdout, stderr=stderr, timeout=timeout, check=False) + item = {"status": "PASS" if proc.returncode == 0 else "FAIL", "exit_code": proc.returncode} + except (OSError, subprocess.TimeoutExpired) as exc: + item = {"status": "FAIL", "detail": type(exc).__name__} + if name == "go_tests": + events = [] + for line in (output / (name + ".stdout.txt")).read_text(errors="replace").splitlines(): + try: + events.append(json.loads(line)) + except ValueError: + pass + counts = Counter(e["Action"] for e in events if "Test" in e and e.get("Action") in ("pass", "skip", "fail")) + item["test_events"] = dict(counts) + item["pending_tests"] = [e["Package"] + "/" + e["Test"] for e in events if e.get("Action") == "skip" and "Test" in e] + if counts["skip"]: + summary["checks"]["go_storage_and_restart_coverage"] = { + "status": "PENDING_INFRA", "detail": "Go skip events are not proof of storage/restart behavior"} + if name == "lab4_scenarios" and (output / "lab4/summary.json").is_file(): + statuses = Counter(v["status"] for v in json.loads((output / "lab4/summary.json").read_text()).values()) + item["case_statuses"] = dict(statuses) + if not statuses["FAIL"] and statuses["PENDING_INFRA"]: + item["status"] = "PENDING_INFRA" + summary["checks"][name] = item + print(item["status"] + " " + name, flush=True) + summary["status"] = "PASS" if all(v["status"] == "PASS" for v in summary["checks"].values()) else "INCOMPLETE" + (output / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n") + print(summary["status"] + ": " + str(output / "summary.json"), flush=True) + return 0 if summary["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main())