From 79e94f9d2a41f20f8e9be1601447e10c280b7bf1 Mon Sep 17 00:00:00 2001 From: idizz Date: Fri, 13 Nov 2026 19:15:04 +0300 Subject: [PATCH 1/2] =?UTF-8?q?S3-=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B8=20=D1=81=D0=B2=D0=B5=D1=80=D0=BA=D0=B0=20?= =?UTF-8?q?Postgresql=20=D1=81=20MinIO?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 31 +- .github/pull_request_template.md | 32 +- Dockerfile | 22 + REPORT.md | 117 +++- compose.yaml | 198 +++++++ deploy/haproxy.cfg | 46 ++ evidence/bootstrap-rerun.txt | 2 + evidence/failover.txt | 6 + evidence/grade45.txt | 24 + evidence/imagecheck.txt | 2 + evidence/jpeg.txt | 2 + evidence/layout.txt | 48 ++ evidence/persistence.txt | 4 + evidence/private-bucket.txt | 4 + evidence/reconcile-apply.json | 53 ++ evidence/restart-after.txt | 6 + evidence/restart-before.txt | 6 + evidence/s3-down.txt | 8 + evidence/s3-rights.txt | 3 + evidence/smoke.txt | 3 + go.mod | 54 ++ go.sum | 122 ++++ infra/README.md | 7 - internal/api/attachments.go | 277 +++++++++ internal/api/auth.go | 182 ++++++ internal/api/handlers.go | 365 ++++++++++++ internal/api/http.go | 338 +++++++++++ internal/api/server.go | 130 +++++ internal/api/server_test.go | 842 +++++++++++++++++++++++++++ internal/blob/blob.go | 32 + internal/blob/memory.go | 86 +++ internal/blob/s3.go | 104 ++++ internal/bootstrap/s3.go | 118 ++++ internal/config/config.go | 169 ++++++ internal/imagecheck/image.go | 190 ++++++ internal/imagecheck/image_test.go | 114 ++++ internal/migrate/migrate.go | 82 +++ internal/password/password.go | 63 ++ internal/password/password_test.go | 23 + internal/reconcile/reconcile.go | 115 ++++ internal/reconcile/reconcile_test.go | 98 ++++ internal/session/redis_test.go | 77 +++ internal/session/session.go | 173 ++++++ internal/store/attachments.go | 286 +++++++++ internal/store/postgres.go | 473 +++++++++++++++ internal/store/postgres_test.go | 183 ++++++ internal/store/store.go | 415 +++++++++++++ messenger/main.go | 309 ++++++++++ migrations/001_init.sql | 50 ++ migrations/002_idempotency.sql | 18 + migrations/003_attachments.sql | 42 ++ migrations/embed.go | 7 + scripts/check_dependencies.sh | 83 +++ scripts/grade4.py | 143 +++++ scripts/restart.sh | 5 + scripts/rolling-update.sh | 28 + scripts/stop-one.sh | 22 + student-tests/grade45.py | 246 ++++++++ student-tests/mkjpeg.go | 18 + student-tests/s3_rights.go | 46 ++ 60 files changed, 6702 insertions(+), 50 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 deploy/haproxy.cfg create mode 100644 evidence/bootstrap-rerun.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/grade45.txt create mode 100644 evidence/imagecheck.txt create mode 100644 evidence/jpeg.txt create mode 100644 evidence/layout.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/private-bucket.txt create mode 100644 evidence/reconcile-apply.json create mode 100644 evidence/restart-after.txt create mode 100644 evidence/restart-before.txt create mode 100644 evidence/s3-down.txt create mode 100644 evidence/s3-rights.txt create mode 100644 evidence/smoke.txt create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 infra/README.md create mode 100644 internal/api/attachments.go create mode 100644 internal/api/auth.go create mode 100644 internal/api/handlers.go create mode 100644 internal/api/http.go create mode 100644 internal/api/server.go create mode 100644 internal/api/server_test.go create mode 100644 internal/blob/blob.go create mode 100644 internal/blob/memory.go create mode 100644 internal/blob/s3.go create mode 100644 internal/bootstrap/s3.go create mode 100644 internal/config/config.go create mode 100644 internal/imagecheck/image.go create mode 100644 internal/imagecheck/image_test.go create mode 100644 internal/migrate/migrate.go create mode 100644 internal/password/password.go create mode 100644 internal/password/password_test.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 internal/reconcile/reconcile_test.go create mode 100644 internal/session/redis_test.go create mode 100644 internal/session/session.go create mode 100644 internal/store/attachments.go create mode 100644 internal/store/postgres.go create mode 100644 internal/store/postgres_test.go create mode 100644 internal/store/store.go create mode 100644 messenger/main.go create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/embed.go create mode 100755 scripts/check_dependencies.sh create mode 100755 scripts/grade4.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/grade45.py create mode 100644 student-tests/mkjpeg.go create mode 100644 student-tests/s3_rights.go diff --git a/.env.example b/.env.example index d99d26a2..b07c382d 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,29 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Скопируйте в .env. Файл .env не коммитится. +# Значения ниже — учебные заглушки локального стенда, не боевые секреты. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me +POSTGRES_DB=messenger +REDIS_PASSWORD=change-me APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +SHUTDOWN_TIMEOUT_SECONDS=8 +API_REPLICAS=2 +API_CPU_LIMIT=0.50 +API_MEM_LIMIT=384M +DB_MAX_CONNS=8 +ARGON2_TIME=2 +ARGON2_MEMORY_KIB=19456 +ARGON2_THREADS=1 +MINIO_ROOT_USER=minio-root +MINIO_ROOT_PASSWORD=change-me-root +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY=messenger-api +S3_SECRET_KEY=change-me-change-me MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +RECONCILE_GRACE_SECONDS=900 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..4baa700c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,22 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Колмычков Иван Сергеевич, БВТ2303 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2 - согласованность PostgreSQL и приватного bucket MinIO -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: из lab3 перенесены PostgreSQL, сессии Redis, две реплики API, HAProxy и идемпотентность сообщений. Добавлены вложения PNG/JPEG: метаданные в PostgreSQL, байты в приватном bucket `messenger-attachments`. Ключ объекта назначает сервер. Загрузка и скачивание идут через API. `201` возвращается после записи в MinIO и в базу; при неудачном INSERT объект удаляется. Сверка `reconcile` убирает старых сирот и старые неприкреплённые строки, не трогая свежие и уже используемые вложения. У API отдельный ключ S3 без прав root. Консоль MinIO наружу не открыта. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build`. Дождаться `GET http://127.0.0.1:8080/health/ready`. Проверки: `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`, `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, `python3 student-tests/grade45.py`. После failover реплика остаётся остановленной: `docker compose --env-file .env start api`. Остановка без удаления данных: `docker compose --env-file .env stop`. Volumes удаляются только явно: `docker compose --env-file .env down -v`. + +Результаты проверок и ссылка на REPORT.md: подробности и таблица критериев — в REPORT.md, выводы команд — в evidence/. `make check` — PASS. `make test` — PASS (`evidence/smoke.txt`). Persistence — PASS, перезапущены API, PostgreSQL, Redis, MinIO и HAProxy (`evidence/persistence.txt`, `evidence/restart-after.txt`). Failover — PASS, 2 реплики, восстановление записи за 2.317 с, байты вложения совпали (`evidence/failover.txt`). Остановленный MinIO даёт 503 на `/health/ready` (`evidence/s3-down.txt`). Анонимный S3 GET — 403, консоль с хоста недоступна (`evidence/private-bucket.txt`). Чужой bucket прикладным ключом — AccessDenied (`evidence/s3-rights.txt`). Негативные загрузки, 10 параллельных загрузок (пик API 71.4 МиБ) и трек 2 — `evidence/grade45.txt`, `evidence/reconcile-apply.json`. + +Ограничения / известные ошибки: один Docker-host не переживает потерю машины. Сверка не восстанавливает пропавшие байты: у используемого вложения без объекта скачивание даёт 404, строка остаётся. Полная распаковка изображения отключена выше 1 000 000 пикселей. Пик памяти измерен на десяти телах по 1 МиБ, не на десяти файлах по 10 МиБ. Образ `docker.io/minio/minio` на Docker Hub отсутствует, поэтому сервер — `alpine/minio:RELEASE.2025-10-15T17-29-55Z` с зафиксированным digest; `minio --version` у этой сборки печатает `DEVELOPMENT.GOGET`. Трек 1 (presigned upload) не делался. + +Источники и использование ИИ: контракт репозитория, MinIO admin API (`madmin-go` v3) и S3 API (`minio-go` v7). Нейросеть помогала писать код, отчёт и прогоны. Секреты в PR нет: пароли только в локальном `.env`. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..604d58f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM golang:1.26-bookworm AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY messenger ./messenger +COPY internal ./internal +COPY migrations ./migrations +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \ + go build -trimpath -ldflags='-s -w' -o /out/messenger ./messenger \ + && chmod 755 /out/messenger + +FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab +COPY --from=build --chown=65532:65532 /out/messenger /messenger +USER 65532:65532 +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + SHUTDOWN_TIMEOUT_SECONDS=8 +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \ + CMD ["/messenger", "healthcheck"] +ENTRYPOINT ["/messenger"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..1a5d7907 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,118 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Колмычков Иван Сергеевич БВТ2303 +- Целевая оценка: **5**. Трек 5: **2, согласованность PostgreSQL и S3**. +- Commit/PR: локальная ветка `solution/bvt2303-kolmychkov`, PR ещё не открыт. +- Источники и помощь ИИ: контракт репозитория, MinIO admin API (`madmin-go` v3) и S3 API (`minio-go` v7). Нейросеть помогала писать код, отчёт и прогоны. Числа ниже сняты с этого компьютера. + +Трек 1 (presigned upload) не выбран: он добавляет второй протокол загрузки, срок подписи и отдельный доступ клиента к bucket. Трек 2 остаётся внутри уже обязательного proxy-upload и проверяется своими командами `reconcile`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверено 2026-11-13 на Linux 7.0.0-31-generic, x86_64, 8 логических CPU, 7.0 ГиБ RAM. Docker Engine 29.1.3, Compose v2.40.3, локальный Go 1.26, Python 3.14. Образы зафиксированы тегами, MinIO ещё и digest. `latest` нет. + +| Роль | service | Образ | Лимит | Порт на хосте | Volume | +| --- | --- | --- | --- | --- | --- | +| API ×2 | api | сборка `messenger-api:lab4` из `Dockerfile` (`golang:1.26-bookworm`, runtime `gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab`) | 0.50 CPU / 384 МиБ на реплику | нет, только через HAProxy | нет | +| PostgreSQL | postgres | `postgres:16.10-bookworm` | 1.00 CPU / 512 МиБ | `127.0.0.1:5432` | `pgdata` | +| Redis | redis | `redis:7.4.6-alpine`, AOF `everysec` | 0.25 CPU / 128 МиБ | `127.0.0.1:6379` | `redisdata` | +| MinIO | minio | `alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4` | 0.50 CPU / 512 МиБ | `127.0.0.1:9000` | `miniodata` | +| Bootstrap | minio-init | тот же образ API, команда `bootstrap-s3` | — | нет | нет | +| HAProxy | haproxy | `haproxy:2.8.14` | 0.25 CPU / 128 МиБ | `127.0.0.1:8080` | нет | + +`docker.io/minio/minio` на эту дату отвечает 404, поэтому сервер взят из Alpine-сборки релиза `RELEASE.2025-10-15T17-29-55Z`. В логе старта виден исходный модуль `github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648`. Строка `minio --version` у этой сборки — `DEVELOPMENT.GOGET`: в образ не передали release ldflags. Закреплены тег и digest. Консоль слушает `127.0.0.1:9001` внутри контейнера и на хост не проброшена. С хоста `curl http://127.0.0.1:9001/` дал код curl 7 (соединения нет). + +```sh +cp .env.example .env +# учебные пароли уже в примере; перед чужой машиной их меняют +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# дождаться /health/ready через http://127.0.0.1:8080 +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# после failover реплика остаётся остановленной: +docker compose --env-file .env start api +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Учебные данные удаляются только явно: `docker compose --env-file .env down -v`. + +Секреты — переменные `.env`, файл не коммитится. Root MinIO есть только у `minio-init`. API получает `S3_ACCESS_KEY` / `S3_SECRET_KEY` и политику `messenger-app`: `ListBucket`, `GetBucketLocation`, `GetObject`, `PutObject`, `DeleteObject` на один bucket. Повтор `bootstrap-s3` не пересоздаёт bucket и не падает, если политика уже привязана. + +Миграции: `001_init.sql`, `002_idempotency.sql`, `003_attachments.sql`. Их применяет одноразовый сервис `migrate`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy → API ×2 → PostgreSQL (пользователи, чаты, сообщения, метаданные вложений) и Redis (сессии). Байты лежат в приватном bucket `messenger-attachments`. Локального файла оригинала на API нет: реплика читает объект по ключу из БД. + +Ключ объекта — `o/` и 32 случайных байта. `filename` только для показа. В JSON ответа нет ключа, адреса bucket и учётных данных. + +`201` возвращается после успешного `PutObject` и `INSERT`. Если `INSERT` не прошёл, процесс сразу вызывает `DeleteObject`. Если процесс умер между этими шагами, объект остаётся сиротой до сверки. Сверка не трогает его, пока не прошёл `RECONCILE_GRACE_SECONDS` (на стенде 900, в эксперименте 5). + +Готовность `/health/ready` требует PostgreSQL, Redis и `BucketExists`. Остановка MinIO даёт 503 на балансировщике, потому что обе реплики перестают быть ready. + +Загрузка читает не больше 10 МиБ + 1 байт. Тип определяется по сигнатуре, не по имени файла. Размеры берутся из IHDR/SOF. Полный `image.Decode` делается только если пикселей не больше 1 000 000. Выше этого порога и до 40 000 000 пикселей достаточно заголовка: растр `width*height*4` не выделяется. Повреждённый PNG/JPEG и подмена MIME дают 415, лишний байт сверх 10 МиБ и слишком большие размеры — 413. Эти ответы не превращаются в 500. + +Пустой текст сообщения допустим только вместе с вложением. Пробельный текст отклоняется. Чужое вложение и вложение другого чата — 404, без сообщения о том, что объект существует. + +Из lab3 перенесены сессии Redis, две реплики, HAProxy без retry POST и идемпотентность сообщения. Отпечаток ключа теперь включает текст и список `attachment_ids`; отсутствующий список равен `[]`. ## Доказательства критериев +Каждая строка — отдельный пункт README. Наблюдение лежит в названном файле, без токенов и паролей. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Возможности lab3 сохранены: сессия, идемпотентность, две реплики, smoke lab4 | `make test`; failover видит число реплик до отказа | Smoke PASS. До отказа наблюдались 2 реплики | `evidence/smoke.txt`, `evidence/failover.txt` | +| Метаданные в PostgreSQL, байты не на диске API | `docker inspect` mounts реплики API | `Mounts` пустой: у API нет volume с оригиналом | `evidence/layout.txt` | +| POST 201 ready, скачивание побайтно, участник читает, посторонний получает 404 | `make test` | PASS: checksum оригинала и 404 для третьего пользователя входят в публичный smoke | `evidence/smoke.txt` | +| Реальный JPEG принимается и скачивается теми же байтами | `go run student-tests/mkjpeg.go`, затем POST/GET через API | 201 `image/jpeg`, `status=ready`, download совпал, 613 байт | `evidence/jpeg.txt` | +| PNG fixture, MIME по содержимому, опасное имя, лимит пикселей до распаковки | `go test ./internal/imagecheck`; `python3 student-tests/grade45.py` | Fixture 320×200 проходит. Имя с `/` и `\` — 400. Текст и подмена MIME — 415. Обрезанный PNG — 415. Файл 10 МиБ+1 и 20000×2001 — 413. Строк вложений на отказах не прибавилось | `evidence/imagecheck.txt`, `evidence/grade45.txt` | +| Upload, остановка обслужившей реплики, download через другую тем же файлом | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS. Тест сам сравнивает байты вложения после остановки реплики. 2 реплики, 120 успешных записей, 1 ошибка перехода, восстановление 2.317 с | `evidence/failover.txt` | +| Restart с volumes сохраняет метаданные и оригинал | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS, включая повторное чтение файла. `StartedAt` сменился у api ×2, postgres, redis, minio, haproxy | `evidence/persistence.txt`, `evidence/restart-before.txt`, `evidence/restart-after.txt` | +| Слишком большой файл, битое изображение, подмена MIME, чужой attachment_id, файл другого чата; не 500 | `python3 student-tests/grade45.py` | 413, 415, 415, 404, 404. В телах нет `internal_error` | `evidence/grade45.txt` | +| Сбой между S3 и PostgreSQL не оставляет вечный мусор; grace | `grade45.py`: сирота и строка без объекта старше 5 с, затем `reconcile` dry-run и `--apply` | Сухой прогон удалил 0. Apply удалил 1 старый объект и 1 старую неприкреплённую строку. Повторный apply удалил 0 | `evidence/grade45.txt`, `evidence/reconcile-apply.json` | +| ≥10 одновременных загрузок, пик памяти ограничен | 10 потоков по 1 МиБ, `docker stats` | 10 ответов 415, пик API 71.4 МиБ при лимите 384 МиБ, 8.43 с | `evidence/grade45.txt` | +| Bootstrap bucket идемпотентен | повторный `docker compose run --rm --no-deps minio-init` | `bootstrap-s3 complete`, код 0, bucket не пересоздавался с ошибкой | `evidence/bootstrap-rerun.txt` | +| Отдельные ограниченные S3 credentials | имена env API и `go run student-tests/s3_rights.go` | У API нет `MINIO_ROOT_*`. Свой bucket виден, `MakeBucket` чужого bucket — `AccessDenied` | `evidence/layout.txt`, `evidence/s3-rights.txt` | +| Консоль не открыта в общей сети | `curl http://127.0.0.1:9001/` и список портов | Код curl 7, HTTP-кода нет. Порт 9001 не проброшен на хост | `evidence/private-bucket.txt`, `evidence/layout.txt` | +| Приватный bucket вне API | `curl` без подписи на `127.0.0.1:9000/messenger-attachments/...` | 403 | `evidence/private-bucket.txt` | +| Недоступность S3 | `docker compose stop minio`, затем `GET /health/ready`, затем `start` | Во время остановки 503 `unavailable`. После старта снова 200 | `evidence/s3-down.txt` | +| Трек 2: ошибки на обеих сторонах, dry-run, повтор, свежие и используемые не удаляются, стоимость обхода | `python3 student-tests/grade45.py` | См. следующий раздел. Свежий сирота остался, метаданные используемого вложения — 200, его байты — 404, свежий файл скачался. Обход: 1 страница листинга, 2 мс и 4 мс | `evidence/grade45.txt`, `evidence/reconcile-apply.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперимент трека 2 -## Эксперименты и отказы +Гипотеза: один проход по множеству ключей bucket и множеству `object_key` в PostgreSQL отличает пару, свежего сироту, старого сироту, свежую строку без объекта, старую строку без объекта и строку, на которую уже ссылается сообщение. Сухой прогон ничего не удаляет. Повторный apply после исправления не удаляет ничего. Свежий сирота и используемое вложение переживают apply. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Методика на том же стенде. Grace для одноразовой команды `reconcile` — 5 секунд, у долгоживущих API остаётся 900. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +1. Загрузить PNG и прикрепить его к сообщению (используемое). +2. Загрузить второй PNG и не прикреплять (кандидат на удаление строки). +3. Положить объект `o/orphan-old` командой `s3-put`. +4. Подождать 6 секунд. +5. Удалить из bucket объекты используемого и неприкреплённого вложений (`s3-rm`). +6. Загрузить ещё один PNG (свежая пара) и положить `o/orphan-fresh`. +7. Два сухих прогона, затем `--apply`, затем `--apply` ещё раз. +8. Проверить HTTP. -## Ограничения и следующие шаги +Наблюдение сухого прогона: 10 строк, 9 объектов, 1 страница листинга, 7 пар, 1 свежий сирота, 1 старый сирота, 2 отсутствующих объекта у используемых строк, 1 старая неприкреплённая строка без объекта. Удалено 0. Второй сухой прогон дал те же счётчики удаления. Две используемые строки — это вложение этого прогона и ещё одно, оставшееся от предыдущей инъекции в том же volume; оба сохранены. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Apply удалил 1 объект и 1 строку за 4 мс. Повторный apply: `deleted_objects=0`, `deleted_rows=0`, старых сирот 0. Свежий сирота остался в классификации (`orphan_fresh=1`). + +HTTP после apply: + +- метаданные используемого вложения — 200; +- его байты — 404, строка не удалена; +- неприкреплённая старая строка — 404; +- свежий файл скачивается побайтно, тело совпало с исходным PNG. + +Стоимость обхода: один `ListObjects` (здесь 1 страница) и один `SELECT` по `attachments`, без `HEAD` на каждый ключ. На 9 объектах сухой прогон занял 2 мс, apply — 4 мс. Это не оценка большого каталога: набор маленький, время сопоставимо с задержкой одного запроса к MinIO на этой машине. + +## Ограничения + +- Один Docker-host. Падение хоста уносит и БД, и MinIO. Две реплики API не делают хранилища высокодоступными. +- Сверка не восстанавливает потерянные байты. Если объект используемого вложения пропал, строка остаётся, скачивание даёт 404. +- Grace защищает свежие сироты, но не вечный мусор: объект старше grace и без строки удаляется. На учебном стенде grace 900 секунд; эксперимент специально ставил 5. +- Полная распаковка растра выключена выше 1 000 000 пикселей. Файл с верным заголовком и битым телом крупнее этого порога может пройти структурную проверку. +- Пик 71.4 МиБ измерен для десяти тел по 1 МиБ. Верхняя оценка буфера — 10 МиБ на запрос, который ещё читается. Десять полных файлов по 10 МиБ в эксперименте не гонялись. +- `alpine/minio` сообщает версию `DEVELOPMENT.GOGET`. Идентичность сборки подтверждается тегом, digest и commit в логе, не строкой `--version`. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..b6f3733a --- /dev/null +++ b/compose.yaml @@ -0,0 +1,198 @@ +# Локальный стенд лабораторной 4. Секреты только из .env, volumes не удаляются restart-скриптом. +# Клиент ходит в haproxy:8080. Реплики api и консоль MinIO наружу не публикуются. +name: messenger-lab4 + +services: + postgres: + image: postgres:16.10-bookworm + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + command: + - postgres + - -c + - synchronous_commit=on + - -c + - max_connections=50 + volumes: + - pgdata:/var/lib/postgresql/data + ports: + - "127.0.0.1:5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + restart: unless-stopped + + redis: + image: redis:7.4.6-alpine + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD} + command: + - sh + - -c + - exec redis-server --appendonly yes --appendfsync everysec --aof-use-rdb-preamble yes --dir /data --requirepass "$$REDIS_PASSWORD" --bind 0.0.0.0 --protected-mode yes + volumes: + - redisdata:/data + ports: + - "127.0.0.1:6379:6379" + healthcheck: + test: ["CMD-SHELL", "redis-cli --no-auth-warning -a \"$$REDIS_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 3s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: unless-stopped + + minio: + # Официальный docker.io/minio/minio на Docker Hub к ноябрю 2026 отдаёт 404. + # Это Alpine-сборка того же сервера, тег и digest зафиксированы. + image: alpine/minio:RELEASE.2025-10-15T17-29-55Z@sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 + # Том создаётся от root. Образ по умолчанию — user minio и не может инициализировать /data. + user: "0:0" + command: ["server", "/data", "--console-address", "127.0.0.1:9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + volumes: + - miniodata:/data + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + restart: unless-stopped + + minio-init: + build: . + image: messenger-api:lab4 + command: ["bootstrap-s3"] + depends_on: + minio: + condition: service_healthy + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_ENDPOINT: http://minio:9000 + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + restart: "no" + + migrate: + build: . + image: messenger-api:lab4 + command: ["migrate"] + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + restart: "no" + + api: + build: . + image: messenger-api:lab4 + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + minio-init: + condition: service_completed_successfully + minio: + condition: service_healthy + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable + REDIS_URL: redis://:${REDIS_PASSWORD}@redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_MAX_CONNS: ${DB_MAX_CONNS:-8} + ARGON2_TIME: ${ARGON2_TIME:-2} + ARGON2_MEMORY_KIB: ${ARGON2_MEMORY_KIB:-19456} + ARGON2_THREADS: ${ARGON2_THREADS:-1} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900} + expose: + - "8080" + healthcheck: + test: ["CMD", "/messenger", "healthcheck"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-0.50}" + memory: ${API_MEM_LIMIT:-384M} + restart: unless-stopped + + haproxy: + image: haproxy:2.8.14 + depends_on: + api: + condition: service_healthy + volumes: + - ./deploy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 5s + timeout: 3s + retries: 6 + start_period: 3s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: unless-stopped + +volumes: + pgdata: + redisdata: + miniodata: diff --git a/deploy/haproxy.cfg b/deploy/haproxy.cfg new file mode 100644 index 00000000..f4f9c83b --- /dev/null +++ b/deploy/haproxy.cfg @@ -0,0 +1,46 @@ +# Единая точка входа. Sticky session нет: сессия лежит в Redis, любая реплика её читает. +# server-template берёт все A-записи DNS Compose (127.0.0.11) и сам выкидывает backend, +# у которого /health/ready перестал отвечать 200. +# +# Обнаружение отказа: inter 1s, fall 2 → не больше 2 с после 503/обрыва. +# hold valid 1s — как часто перечитываем адреса реплик после scale/stop. +# retries 0: балансировщик не повторяет POST. Повтор сообщения делает клиент тем же ключом. +# Создание пользователя и чата не идемпотентно, поэтому прозрачный retry на LB запрещён. + +global + maxconn 256 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + accepted_payload_size 8192 + +defaults + mode http + timeout connect 2s + timeout client 120s + timeout server 120s + timeout http-request 10s + timeout check 2s + timeout queue 2s + retries 0 + +frontend http_in + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 1 maxconn 64 + server-template api 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 diff --git a/evidence/bootstrap-rerun.txt b/evidence/bootstrap-rerun.txt new file mode 100644 index 00000000..a2684cbe --- /dev/null +++ b/evidence/bootstrap-rerun.txt @@ -0,0 +1,2 @@ +2026/11/13 15:21:03 INFO bootstrap-s3 complete +bootstrap_exit:0 diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..a98f7306 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,6 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 1, "first_recovery_seconds": 2.317} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +failover_exit:0 +ready_after_restore 200 diff --git a/evidence/grade45.txt b/evidence/grade45.txt new file mode 100644 index 00000000..def98446 --- /dev/null +++ b/evidence/grade45.txt @@ -0,0 +1,24 @@ +slash filename: HTTP 400 expected 400 PASS body=b'{"error":{"code":"invalid_request","message":"\xd0\x9d\xd0\xb5\xd0\xba\xd0\xbe\xd1\x80\xd1\x80\xd0\xb5\xd0\xba\xd1\x82\xd0\xbd\xd0\xbe\xd0\xb5 \xd0\xb8\xd0\xbc\xd1\x8f \xd1\x84\xd0\xb0\xd0\xb9\xd0\xbb\xd0\xb0","request_id":"f87e88bb3a77dfcd1954869b6bd7cd1f"}}\n' +backslash filename: HTTP 400 expected 400 PASS body=b'{"error":{"code":"invalid_request","message":"\xd0\x9d\xd0\xb5\xd0\xba\xd0\xbe\xd1\x80\xd1\x80\xd0\xb5\xd0\xba\xd1\x82\xd0\xbd\xd0\xbe\xd0\xb5 \xd0\xb8\xd0\xbc\xd1\x8f \xd1\x84\xd0\xb0\xd0\xb9\xd0\xbb\xd0\xb0","request_id":"7331f9440909ae6caf0fb44ee96e9f32"}}\n' +text as png: HTTP 415 expected 415 PASS body=b'{"error":{"code":"unsupported_media_type","message":"\xd0\xa4\xd0\xb0\xd0\xb9\xd0\xbb \xd0\xbd\xd0\xb5 \xd1\x8f\xd0\xb2\xd0\xbb\xd1\x8f\xd0\xb5\xd1\x82\xd1\x81\xd1\x8f PNG \xd0\xb8\xd0\xbb\xd0\xb8 JPEG","request_id":"0dce761035b9400dd19b242221eb8481"}}\n' +mime swap: HTTP 415 expected 415 PASS body=b'{"error":{"code":"unsupported_media_type","message":"\xd0\xa4\xd0\xb0\xd0\xb9\xd0\xbb \xd0\xbd\xd0\xb5 \xd1\x8f\xd0\xb2\xd0\xbb\xd1\x8f\xd0\xb5\xd1\x82\xd1\x81\xd1\x8f PNG \xd0\xb8\xd0\xbb\xd0\xb8 JPEG","request_id":"89eae50ae499ffa84222ea1b9fd09a18"}}\n' +truncated png: HTTP 415 expected 415 PASS body=b'{"error":{"code":"unsupported_media_type","message":"\xd0\xa4\xd0\xb0\xd0\xb9\xd0\xbb \xd0\xbd\xd0\xb5 \xd1\x8f\xd0\xb2\xd0\xbb\xd1\x8f\xd0\xb5\xd1\x82\xd1\x81\xd1\x8f PNG \xd0\xb8\xd0\xbb\xd0\xb8 JPEG","request_id":"2cb180557a19a4a557ed8df5b1da7308"}}\n' +over 10MiB: HTTP 413 expected 413 PASS body=b'{"error":{"code":"payload_too_large","message":"\xd0\xa4\xd0\xb0\xd0\xb9\xd0\xbb \xd0\xb1\xd0\xbe\xd0\xbb\xd1\x8c\xd1\x88\xd0\xb5 10 \xd0\x9c\xd0\xb8\xd0\x91","request_id":"c157c5c1d8ec893fbb5d1f173765ec58"}}\n' +pixel bomb: HTTP 413 expected 413 PASS body=b'{"error":{"code":"payload_too_large","message":"\xd0\xa1\xd0\xbb\xd0\xb8\xd1\x88\xd0\xba\xd0\xbe\xd0\xbc \xd0\xbc\xd0\xbd\xd0\xbe\xd0\xb3\xd0\xbe \xd0\xbf\xd0\xb8\xd0\xba\xd1\x81\xd0\xb5\xd0\xbb\xd0\xb5\xd0\xb9","request_id":"83373190a90631c1480641d4a50fce7c"}}\n' +rejected uploads left attachment rows 5->5 PASS +good upload: HTTP 201 expected 201 PASS body=b'{"id":"3c272598-75cc-417b-9277-3049201482be","chat_id":"429de0c2-a4a9-4187-a741-967924abe036","owner_id":"7fc52156-5f47-4166-8326-20e277d73026","filename":"ok.png","content_type":"' +unknown attachment: HTTP 404 expected 404 PASS body=b'{"error":{"code":"not_found","message":"\xd0\x9e\xd0\xb1\xd1\x8a\xd0\xb5\xd0\xba\xd1\x82 \xd0\xbd\xd0\xb5 \xd0\xbd\xd0\xb0\xd0\xb9\xd0\xb4\xd0\xb5\xd0\xbd","request_id":"f77f64b8660f9bb0a953e7e11edfc927"}}\n' +other chat upload: HTTP 201 expected 201 PASS body=b'{"id":"f1cb80e3-3359-43e9-8f9a-b94aeef3aa62","chat_id":"f5e6ff73-8216-4083-b669-c70d9114b9ec","owner_id":"7fc52156-5f47-4166-8326-20e277d73026","filename":"foreign.png","content_ty' +attachment from other chat: HTTP 404 expected 404 PASS body=b'{"error":{"code":"not_found","message":"\xd0\x9e\xd0\xb1\xd1\x8a\xd0\xb5\xd0\xba\xd1\x82 \xd0\xbd\xd0\xb5 \xd0\xbd\xd0\xb0\xd0\xb9\xd0\xb4\xd0\xb5\xd0\xbd","request_id":"e0c7125bce6ab391a87be26308fbe587"}}\n' +outsider download: HTTP 404 expected 404 PASS body=b'{"error":{"code":"not_found","message":"\xd0\x9e\xd0\xb1\xd1\x8a\xd0\xb5\xd0\xba\xd1\x82 \xd0\xbd\xd0\xb5 \xd0\xbd\xd0\xb0\xd0\xb9\xd0\xb4\xd0\xb5\xd0\xbd","request_id":"f5d96f39fe154172be91ef72756666ec"}}\n' +anonymous S3 GET 403 PASS +console on host http_code=000 err=7 +10 parallel 1MiB uploads statuses=[415, 415, 415, 415, 415, 415, 415, 415, 415, 415] seconds=8.43 peak_api_MiB=71.4 +dry-run-1 {"mode": "dry-run", "grace_seconds": 5, "cutoff": "2026-11-13T13:56:30Z", "db_rows": 10, "s3_objects": 9, "list_pages": 1, "pairs": 7, "orphan_fresh": 1, "orphan_stale": 1, "missing_fresh": 0, "missing_referenced": 2, "missing_stale_unreferenced": 1, "deleted_objects": 0, "deleted_rows": 0, "duration_ms": 2} +dry-run-2 same=True +apply {"mode": "apply", "grace_seconds": 5, "cutoff": "2026-11-13T13:56:31Z", "db_rows": 10, "s3_objects": 9, "list_pages": 1, "pairs": 7, "orphan_fresh": 1, "orphan_stale": 1, "missing_fresh": 0, "missing_referenced": 2, "missing_stale_unreferenced": 1, "deleted_objects": 1, "deleted_rows": 1, "duration_ms": 4} +apply-again {"deleted_objects": 0, "deleted_rows": 0, "orphan_stale": 0, "missing_stale_unreferenced": 0} +referenced metadata after apply HTTP 200 +referenced bytes missing HTTP 404 +stale unreferenced metadata HTTP 404 +fresh download HTTP 200 same_bytes=True diff --git a/evidence/imagecheck.txt b/evidence/imagecheck.txt new file mode 100644 index 00000000..a03cc516 --- /dev/null +++ b/evidence/imagecheck.txt @@ -0,0 +1,2 @@ +ok messenger/internal/imagecheck 0.008s +imagecheck_exit:0 diff --git a/evidence/jpeg.txt b/evidence/jpeg.txt new file mode 100644 index 00000000..729a4e7a --- /dev/null +++ b/evidence/jpeg.txt @@ -0,0 +1,2 @@ +upload HTTP 201 status=ready content_type=image/jpeg size=613 sha256=c4667e8441372971150fbd56d2eed48347fa033801754f1f6c563bf4f45b99da +download HTTP 200 content_type=image/jpeg same_bytes=True len=613 diff --git a/evidence/layout.txt b/evidence/layout.txt new file mode 100644 index 00000000..2d549d1a --- /dev/null +++ b/evidence/layout.txt @@ -0,0 +1,48 @@ +api env names: + +APP_ENV +APP_HOST +APP_PORT +ARGON2_MEMORY_KIB +ARGON2_THREADS +ARGON2_TIME +DATABASE_URL +DB_MAX_CONNS +IDEMPOTENCY_TTL_SECONDS +MAX_IMAGE_PIXELS +MAX_UPLOAD_BYTES +PATH +RECONCILE_GRACE_SECONDS +REDIS_URL +S3_ACCESS_KEY +S3_BUCKET +S3_ENDPOINT +S3_REGION +S3_SECRET_KEY +SESSION_TTL_SECONDS +SHUTDOWN_TIMEOUT_SECONDS +SSL_CERT_FILE +api mounts: +[] +minio-init env names: + +APP_ENV +APP_HOST +APP_PORT +MINIO_ROOT_PASSWORD +MINIO_ROOT_USER +PATH +S3_ACCESS_KEY +S3_BUCKET +S3_ENDPOINT +S3_REGION +S3_SECRET_KEY +SHUTDOWN_TIMEOUT_SECONDS +SSL_CERT_FILE +published ports: +api 8080/tcp +api 8080/tcp +haproxy 127.0.0.1:8080->8080/tcp +minio 127.0.0.1:9000->9000/tcp, 9001/tcp +postgres 127.0.0.1:5432->5432/tcp +redis 127.0.0.1:6379->6379/tcp diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..7940e23b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +persistence_exit:0 diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..d97356ec --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,4 @@ +console host: +http_code=000 exit=7 +anonymous object: +http_code=403 diff --git a/evidence/reconcile-apply.json b/evidence/reconcile-apply.json new file mode 100644 index 00000000..485035fd --- /dev/null +++ b/evidence/reconcile-apply.json @@ -0,0 +1,53 @@ +{ + "dry": { + "mode": "dry-run", + "grace_seconds": 5, + "cutoff": "2026-11-13T13:56:30Z", + "db_rows": 10, + "s3_objects": 9, + "list_pages": 1, + "pairs": 7, + "orphan_fresh": 1, + "orphan_stale": 1, + "missing_fresh": 0, + "missing_referenced": 2, + "missing_stale_unreferenced": 1, + "deleted_objects": 0, + "deleted_rows": 0, + "duration_ms": 2 + }, + "apply": { + "mode": "apply", + "grace_seconds": 5, + "cutoff": "2026-11-13T13:56:31Z", + "db_rows": 10, + "s3_objects": 9, + "list_pages": 1, + "pairs": 7, + "orphan_fresh": 1, + "orphan_stale": 1, + "missing_fresh": 0, + "missing_referenced": 2, + "missing_stale_unreferenced": 1, + "deleted_objects": 1, + "deleted_rows": 1, + "duration_ms": 4 + }, + "again": { + "mode": "apply", + "grace_seconds": 5, + "cutoff": "2026-11-13T13:56:31Z", + "db_rows": 9, + "s3_objects": 8, + "list_pages": 1, + "pairs": 7, + "orphan_fresh": 1, + "orphan_stale": 0, + "missing_fresh": 0, + "missing_referenced": 2, + "missing_stale_unreferenced": 0, + "deleted_objects": 0, + "deleted_rows": 0, + "duration_ms": 2 + } +} diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 00000000..e1ff3439 --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,6 @@ +/messenger-lab4-api-1 2026-11-13T15:21:24.371344153Z changed=True +/messenger-lab4-api-2 2026-11-13T15:21:24.097492187Z changed=True +/messenger-lab4-haproxy-1 2026-11-13T15:21:23.853867151Z changed=True +/messenger-lab4-minio-1 2026-11-13T15:21:23.912731663Z changed=True +/messenger-lab4-postgres-1 2026-11-13T15:21:23.929193078Z changed=True +/messenger-lab4-redis-1 2026-11-13T15:21:23.798144563Z changed=True diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 00000000..d96304d6 --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,6 @@ +/messenger-lab4-api-1 2026-11-13T13:57:38.601482122Z +/messenger-lab4-api-2 2026-11-13T13:57:12.678628253Z +/messenger-lab4-haproxy-1 2026-11-13T13:57:12.296072145Z +/messenger-lab4-minio-1 2026-11-13T13:57:12.399188879Z +/messenger-lab4-postgres-1 2026-11-13T13:57:12.441137115Z +/messenger-lab4-redis-1 2026-11-13T13:57:12.342361107Z diff --git a/evidence/s3-down.txt b/evidence/s3-down.txt new file mode 100644 index 00000000..892e0eef --- /dev/null +++ b/evidence/s3-down.txt @@ -0,0 +1,8 @@ + Container messenger-lab4-minio-1 Stopping + Container messenger-lab4-minio-1 Stopped +ready_while_stopped {"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"ee574af75872b15d575a895357e5bd95"}} + http=503 + Container messenger-lab4-minio-1 Starting + Container messenger-lab4-minio-1 Started +ready_after_start 503 +ready_after_start 200 diff --git a/evidence/s3-rights.txt b/evidence/s3-rights.txt new file mode 100644 index 00000000..b984dc36 --- /dev/null +++ b/evidence/s3-rights.txt @@ -0,0 +1,3 @@ +BucketExists messenger-attachments: ok=true no_error=true +MakeBucket messenger-other-bucket: AccessDenied +rights_exit:0 diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..81b8a927 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,3 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. +smoke_exit:0 diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..7a647786 --- /dev/null +++ b/go.mod @@ -0,0 +1,54 @@ +module messenger + +go 1.26.0 + +require ( + github.com/jackc/pgx/v5 v5.11.0 + github.com/redis/go-redis/v9 v9.23.0 + golang.org/x/crypto v0.58.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/crc64nvme v1.0.2 // indirect + github.com/minio/madmin-go/v3 v3.0.110 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.95 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.63.0 // indirect + github.com/prometheus/procfs v0.16.0 // indirect + github.com/prometheus/prom2json v1.4.2 // indirect + github.com/prometheus/prometheus v0.303.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/safchain/ethtool v0.5.10 // indirect + github.com/secure-io/sio-go v0.3.1 // indirect + github.com/shirou/gopsutil/v3 v3.24.5 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/tinylib/msgp v1.3.0 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.uber.org/atomic v1.12.0 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..70d49d48 --- /dev/null +++ b/go.sum @@ -0,0 +1,122 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg= +github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/madmin-go/v3 v3.0.110 h1:FIYekj7YPc430ffpXFWiUtyut3qBt/unIAcDzJn9H5M= +github.com/minio/madmin-go/v3 v3.0.110/go.mod h1:WOe2kYmYl1OIlY2DSRHVQ8j1v4OItARQ6jGyQqcCud8= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= +github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= +github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= +github.com/prometheus/prom2json v1.4.2 h1:PxCTM+Whqi/eykO1MKsEL0p/zMpxp9ybpsmdFamw6po= +github.com/prometheus/prom2json v1.4.2/go.mod h1:zuvPm7u3epZSbXPWHny6G+o8ETgu6eAK3oPr6yFkRWE= +github.com/prometheus/prometheus v0.303.0 h1:wsNNsbd4EycMCphYnTmNY9JASBVbp7NWwJna857cGpA= +github.com/prometheus/prometheus v0.303.0/go.mod h1:8PMRi+Fk1WzopMDeb0/6hbNs9nV6zgySkU/zds5Lu3o= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/safchain/ethtool v0.5.10 h1:Im294gZtuf4pSGJRAOGKaASNi3wMeFaGaWuSaomedpc= +github.com/safchain/ethtool v0.5.10/go.mod h1:w9jh2Lx7YBR4UwzLkzCmWl85UY0W2uZdd7/DckVE5+c= +github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= +github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww= +github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/infra/README.md b/infra/README.md deleted file mode 100644 index 5a39e862..00000000 --- a/infra/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Инфраструктура студента - -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. - -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. diff --git a/internal/api/attachments.go b/internal/api/attachments.go new file mode 100644 index 00000000..90832cdb --- /dev/null +++ b/internal/api/attachments.go @@ -0,0 +1,277 @@ +package api + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + "net/http" + "strconv" + "strings" + "unicode/utf8" + + "messenger/internal/blob" + "messenger/internal/imagecheck" + "messenger/internal/store" +) + +type attachmentJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []struct{} `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +func (a *API) routeAttachment(w http.ResponseWriter, r *http.Request, rest string) { + if _, ok := a.authenticate(w, r); !ok { + return + } + rest = strings.Trim(rest, "/") + if rest == "" || strings.Count(rest, "/") > 1 { + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + return + } + id, content, hasContent := strings.Cut(rest, "/") + if hasContent && content != "content" { + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + return + } + if _, ok := canonicalUUID(id); !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный идентификатор", 0) + return + } + if r.Method != http.MethodGet { + a.fail(w, r, http.StatusMethodNotAllowed, "invalid_request", "Метод не поддерживается", 0) + return + } + if hasContent { + a.downloadAttachment(w, r, id) + return + } + a.getAttachment(w, r, id) +} + +func (a *API) uploadAttachment(w http.ResponseWriter, r *http.Request, rawChatID string) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + if a.blobs == nil { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + chatID, ok := a.chatID(w, r, rawChatID) + if !ok { + return + } + if _, err := a.store.ChatFor(r.Context(), chatID, user.ID); a.writeStoreErr(w, r, err) { + return + } + filename := r.URL.Query().Get("filename") + if !validFilename(filename) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректное имя файла", 0) + return + } + media := "" + if ct := r.Header.Get("Content-Type"); ct != "" { + media = strings.ToLower(strings.TrimSpace(strings.Split(ct, ";")[0])) + } + if media != "image/png" && media != "image/jpeg" { + a.fail(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "Нужен PNG или JPEG", 0) + return + } + // Не отвечаем до чтения: иначе клиент ещё шлёт тело и видит обрыв вместо 413. + // Читаем не больше лимита плюс один байт. Лишний байт - отказ, без записи в bucket. + body, err := io.ReadAll(io.LimitReader(r.Body, a.maxUpload+1)) + if err != nil { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Загрузка прервана", 0) + return + } + if int64(len(body)) > a.maxUpload { + a.fail(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "Файл больше 10 МиБ", 0) + return + } + if len(body) == 0 { + a.fail(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "Пустое изображение", 0) + return + } + width, height, err := imagecheck.Inspect(media, body, a.maxPixels) + if err != nil { + if errors.Is(err, imagecheck.ErrPixels) { + a.fail(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "Слишком много пикселей", 0) + return + } + a.fail(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "Файл не является PNG или JPEG", 0) + return + } + sum := sha256.Sum256(body) + key, err := newObjectKey() + if err != nil { + a.fail(w, r, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", 0) + return + } + if err := a.blobs.Put(r.Context(), key, media, body); err != nil { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + row, err := a.store.InsertAttachment(r.Context(), store.Attachment{ + ChatID: chatID, + OwnerID: user.ID, + Filename: filename, + ContentType: media, + SizeBytes: int64(len(body)), + SHA256: hex.EncodeToString(sum[:]), + ObjectKey: key, + Width: width, + Height: height, + }) + if err != nil { + // PUT уже прошёл, COMMIT нет: сразу убираем объект. Если удаление не удалось, + // ключ останется сиротой до сверки и не попадёт в ответ 201. + if delErr := a.blobs.Delete(r.Context(), key); delErr != nil { + a.log.Error("orphan object", "request_id", requestIDFrom(r)) + } + if a.writeStoreErr(w, r, err) { + return + } + } + writeJSON(w, http.StatusCreated, asAttachment(row)) +} + +func (a *API) getAttachment(w http.ResponseWriter, r *http.Request, rawID string) { + row, ok := a.visibleAttachment(w, r, rawID) + if !ok { + return + } + writeJSON(w, http.StatusOK, asAttachment(row)) +} + +func (a *API) downloadAttachment(w http.ResponseWriter, r *http.Request, rawID string) { + row, ok := a.visibleAttachment(w, r, rawID) + if !ok { + return + } + variant := "original" + if r.URL.Query().Has("variant") { + variant = r.URL.Query().Get("variant") + } + switch variant { + case "original": + case "thumbnail", "crop", "optimized": + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + return + default: + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный вариант", 0) + return + } + if a.blobs == nil { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + rc, err := a.blobs.Get(r.Context(), row.ObjectKey) + if err != nil { + if errors.Is(err, blob.ErrNotFound) { + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + return + } + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + defer rc.Close() + w.Header().Set("Content-Type", row.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(row.SizeBytes, 10)) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, rc); err != nil { + a.log.Error("download", "request_id", requestIDFrom(r)) + } +} + +func (a *API) visibleAttachment(w http.ResponseWriter, r *http.Request, rawID string) (store.Attachment, bool) { + user, ok := a.authenticate(w, r) + if !ok { + return store.Attachment{}, false + } + id, ok := canonicalUUID(rawID) + if !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный идентификатор", 0) + return store.Attachment{}, false + } + row, err := a.store.AttachmentVisible(r.Context(), id, user.ID) + if a.writeStoreErr(w, r, err) { + return store.Attachment{}, false + } + return row, true +} + +func (a *API) attachmentIDList(w http.ResponseWriter, r *http.Request, raw []string) ([]string, bool) { + if len(raw) > 10 { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Слишком много вложений", 0) + return nil, false + } + seen := make(map[string]struct{}, len(raw)) + out := make([]string, 0, len(raw)) + for _, item := range raw { + id, ok := canonicalUUID(item) + if !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный идентификатор вложения", 0) + return nil, false + } + if _, dup := seen[id]; dup { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Повторяющееся вложение", 0) + return nil, false + } + seen[id] = struct{}{} + out = append(out, id) + } + return out, true +} + +func asAttachment(row store.Attachment) attachmentJSON { + return attachmentJSON{ + ID: row.ID, + ChatID: row.ChatID, + OwnerID: row.OwnerID, + Filename: row.Filename, + ContentType: row.ContentType, + SizeBytes: row.SizeBytes, + SHA256: row.SHA256, + Status: row.Status, + Variants: []struct{}{}, + ErrorCode: nil, + CreatedAt: formatTime(row.CreatedAt), + UpdatedAt: formatTime(row.UpdatedAt), + } +} + +func validFilename(name string) bool { + if !utf8.ValidString(name) { + return false + } + n := utf8.RuneCountInString(name) + if n < 1 || n > 255 { + return false + } + for _, r := range name { + if r < 0x20 || r == 0x7f || r == '/' || r == '\\' { + return false + } + } + return true +} + +func newObjectKey() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return "o/" + hex.EncodeToString(b[:]), nil +} diff --git a/internal/api/auth.go b/internal/api/auth.go new file mode 100644 index 00000000..9b635bf8 --- /dev/null +++ b/internal/api/auth.go @@ -0,0 +1,182 @@ +package api + +import ( + "crypto/rand" + "encoding/base64" + "encoding/hex" + "errors" + "io" + "net/http" + "strings" + "time" + "unicode/utf8" + + "messenger/internal/password" + "messenger/internal/session" + "messenger/internal/store" +) + +const ( + basicChallenge = `Basic realm="messenger", charset="UTF-8"` + bearerChallenge = `Bearer realm="messenger"` +) + +func (a *API) createSession(w http.ResponseWriter, r *http.Request) { + if !emptyBody(r) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "У входа не должно быть тела", 0) + return + } + username, plain, ok := parseBasic(r) + if !ok || !usernameRe.MatchString(username) || !validPassword(plain) { + a.unauthorized(w, r, basicChallenge) + return + } + user, hash, err := a.store.UserByUsername(r.Context(), username) + if errors.Is(err, store.ErrNotFound) { + password.Verify(plain, a.dummyHash) + a.unauthorized(w, r, basicChallenge) + return + } + if a.writeStoreErr(w, r, err) { + return + } + if !password.Verify(plain, hash) { + a.unauthorized(w, r, basicChallenge) + return + } + token, err := newToken() + if err != nil { + a.fail(w, r, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", 0) + return + } + expires := time.Now().UTC().Add(a.sessionTTL).Truncate(time.Microsecond) + if err := a.sessions.Create(r.Context(), token, user.ID, expires); a.writeStoreErr(w, r, err) { + return + } + writeJSON(w, http.StatusCreated, map[string]any{ + "token": token, + "token_type": "Bearer", + "expires_at": formatTime(expires), + "user": asUser(user), + }) +} + +func (a *API) revokeSession(w http.ResponseWriter, r *http.Request) { + token, ok := a.bearerToken(w, r) + if !ok { + return + } + if _, _, err := a.sessions.Lookup(r.Context(), token); err != nil { + if errors.Is(err, session.ErrNotFound) { + a.unauthorized(w, r, bearerChallenge) + return + } + a.writeStoreErr(w, r, err) + return + } + if err := a.sessions.Revoke(r.Context(), token); err != nil { + if errors.Is(err, session.ErrNotFound) { + a.unauthorized(w, r, bearerChallenge) + return + } + a.writeStoreErr(w, r, err) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (a *API) authenticate(w http.ResponseWriter, r *http.Request) (store.User, bool) { + token, ok := a.bearerToken(w, r) + if !ok { + return store.User{}, false + } + userID, expires, err := a.sessions.Lookup(r.Context(), token) + if errors.Is(err, session.ErrNotFound) || (err == nil && !time.Now().Before(expires)) { + if err == nil { + _ = a.sessions.Revoke(r.Context(), token) + } + a.unauthorized(w, r, bearerChallenge) + return store.User{}, false + } + if err != nil { + a.writeStoreErr(w, r, err) + return store.User{}, false + } + user, err := a.store.User(r.Context(), userID) + if errors.Is(err, store.ErrNotFound) { + a.unauthorized(w, r, bearerChallenge) + return store.User{}, false + } + if a.writeStoreErr(w, r, err) { + return store.User{}, false + } + return user, true +} + +func (a *API) bearerToken(w http.ResponseWriter, r *http.Request) (string, bool) { + scheme, param, ok := splitAuth(r.Header.Get("Authorization")) + if !ok || !strings.EqualFold(scheme, "Bearer") || param == "" || len(param) > 512 || strings.Contains(param, " ") { + a.unauthorized(w, r, bearerChallenge) + return "", false + } + return param, true +} + +func (a *API) unauthorized(w http.ResponseWriter, r *http.Request, challenge string) { + w.Header().Set("WWW-Authenticate", challenge) + a.fail(w, r, http.StatusUnauthorized, "unauthorized", "Нет действительной идентификации", 0) +} + +func parseBasic(r *http.Request) (string, string, bool) { + scheme, param, ok := splitAuth(r.Header.Get("Authorization")) + if !ok || !strings.EqualFold(scheme, "Basic") || param == "" { + return "", "", false + } + raw, err := base64.StdEncoding.DecodeString(param) + if err != nil { + raw, err = base64.RawStdEncoding.DecodeString(param) + if err != nil { + return "", "", false + } + } + if !utf8.Valid(raw) { + return "", "", false + } + username, plain, ok := strings.Cut(string(raw), ":") + if !ok || username == "" { + return "", "", false + } + return username, plain, true +} + +func splitAuth(header string) (string, string, bool) { + scheme, param, ok := strings.Cut(strings.TrimSpace(header), " ") + if !ok { + return "", "", false + } + param = strings.TrimSpace(param) + if scheme == "" || param == "" { + return "", "", false + } + return scheme, param, true +} + +func emptyBody(r *http.Request) bool { + if r.Body == nil || r.ContentLength == 0 { + return true + } + buf := make([]byte, 1) + n, err := r.Body.Read(buf) + if n > 0 { + return false + } + return err == io.EOF || err == nil +} + +func newToken() (string, error) { + var b [32]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} diff --git a/internal/api/handlers.go b/internal/api/handlers.go new file mode 100644 index 00000000..2f407931 --- /dev/null +++ b/internal/api/handlers.go @@ -0,0 +1,365 @@ +package api + +import ( + "context" + "errors" + "net/http" + "strings" + "time" + + "messenger/internal/session" + "messenger/internal/store" +) + +func (a *API) live(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (a *API) ready(w http.ResponseWriter, r *http.Request) { + if a.shutting.Load() { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервер останавливается", 1) + return + } + ctx, cancel := context.WithTimeout(r.Context(), 800*time.Millisecond) + defer cancel() + if err := a.store.Ping(ctx); err != nil || a.sessions.Ping(ctx) != nil { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + if a.blobs != nil { + if err := a.blobs.Ping(ctx); err != nil { + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + return + } + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (a *API) route(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case path == "/api/v1/users" && r.Method == http.MethodPost: + a.createUser(w, r) + case path == "/api/v1/users/me" && r.Method == http.MethodGet: + a.currentUser(w, r) + case path == "/api/v1/auth/sessions" && r.Method == http.MethodPost: + a.createSession(w, r) + case path == "/api/v1/auth/sessions/current" && r.Method == http.MethodDelete: + a.revokeSession(w, r) + case path == "/api/v1/chats" && r.Method == http.MethodPost: + a.createChat(w, r) + case path == "/api/v1/chats" && r.Method == http.MethodGet: + a.listChats(w, r) + default: + if strings.HasPrefix(path, "/api/v1/attachments/") { + a.routeAttachment(w, r, strings.TrimPrefix(path, "/api/v1/attachments/")) + return + } + chatID, kind, ok := splitChatPath(path) + if !ok { + if knownWrongMethod(path) { + a.fail(w, r, http.StatusMethodNotAllowed, "invalid_request", "Метод не поддерживается", 0) + return + } + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + return + } + switch kind { + case "": + if r.Method == http.MethodGet { + a.getChat(w, r, chatID) + return + } + case "messages": + switch r.Method { + case http.MethodPost: + a.createMessage(w, r, chatID) + return + case http.MethodGet: + a.listMessages(w, r, chatID) + return + } + case "attachments": + if r.Method == http.MethodPost { + a.uploadAttachment(w, r, chatID) + return + } + } + a.fail(w, r, http.StatusMethodNotAllowed, "invalid_request", "Метод не поддерживается", 0) + } +} + +func knownWrongMethod(path string) bool { + switch path { + case "/api/v1/users", "/api/v1/users/me", "/api/v1/chats", + "/api/v1/auth/sessions", "/api/v1/auth/sessions/current": + return true + default: + return false + } +} + +func (a *API) createUser(w http.ResponseWriter, r *http.Request) { + var body struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` + } + if !a.readJSON(w, r, &body) { + return + } + if !usernameRe.MatchString(body.Username) || !validText(body.DisplayName, 1, 100) || !validPassword(body.Password) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректные поля пользователя", 0) + return + } + hash, err := a.hasher.Hash(body.Password) + if err != nil { + a.fail(w, r, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", 0) + return + } + user, err := a.store.CreateUser(r.Context(), body.Username, body.DisplayName, hash) + if a.writeStoreErr(w, r, err) { + return + } + writeJSON(w, http.StatusCreated, asUser(user)) +} + +func (a *API) currentUser(w http.ResponseWriter, r *http.Request) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + writeJSON(w, http.StatusOK, asUser(user)) +} + +func (a *API) createChat(w http.ResponseWriter, r *http.Request) { + caller, ok := a.authenticate(w, r) + if !ok { + return + } + var body struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if !validText(body.Title, 1, 100) || len(body.MemberIDs) < 1 || len(body.MemberIDs) > 100 { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректные поля чата", 0) + return + } + ids := make([]string, len(body.MemberIDs)) + seen := make(map[string]struct{}, len(body.MemberIDs)) + callerIn := false + for i, raw := range body.MemberIDs { + id, ok := canonicalUUID(raw) + if !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный список участников", 0) + return + } + if _, dup := seen[id]; dup { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный список участников", 0) + return + } + seen[id] = struct{}{} + _, err := a.store.User(r.Context(), id) + if errors.Is(err, store.ErrNotFound) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный список участников", 0) + return + } + if a.writeStoreErr(w, r, err) { + return + } + if id == caller.ID { + callerIn = true + } + ids[i] = id + } + if !callerIn { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный список участников", 0) + return + } + chat, err := a.store.CreateChat(r.Context(), body.Title, ids, caller.ID) + if errors.Is(err, store.ErrInvalid) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный список участников", 0) + return + } + if a.writeStoreErr(w, r, err) { + return + } + writeJSON(w, http.StatusCreated, asChat(chat)) +} + +func (a *API) listChats(w http.ResponseWriter, r *http.Request) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + limit, cur, ok := a.pageQuery(w, r, "chats:"+user.ID) + if !ok { + return + } + items, hasMore, err := a.store.ListChats(r.Context(), user.ID, cur, limit) + if a.writeStoreErr(w, r, err) { + return + } + out := make([]chatJSON, 0, len(items)) + for _, c := range items { + out = append(out, asChat(c)) + } + var next *string + if hasMore && len(items) > 0 { + last := items[len(items)-1] + encoded := encodeCursor(cursorPayload{Scope: "chats:" + user.ID, ID: last.ID, At: formatTime(last.CreatedAt)}) + next = &encoded + } + writeJSON(w, http.StatusOK, chatPage{Items: out, NextCursor: next}) +} + +func (a *API) getChat(w http.ResponseWriter, r *http.Request, rawID string) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + id, ok := a.chatID(w, r, rawID) + if !ok { + return + } + chat, err := a.store.ChatFor(r.Context(), id, user.ID) + if a.writeStoreErr(w, r, err) { + return + } + writeJSON(w, http.StatusOK, asChat(chat)) +} + +func (a *API) createMessage(w http.ResponseWriter, r *http.Request, rawID string) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + id, ok := a.chatID(w, r, rawID) + if !ok { + return + } + if _, err := a.store.ChatFor(r.Context(), id, user.ID); a.writeStoreErr(w, r, err) { + return + } + idemKey := r.Header.Get("Idempotency-Key") + if !idemKeyRe.MatchString(idemKey) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Нужен заголовок Idempotency-Key", 0) + return + } + var body struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + ids, ok := a.attachmentIDList(w, r, body.AttachmentIDs) + if !ok { + return + } + if !validMessageText(body.Text, len(ids)) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный текст сообщения", 0) + return + } + // Отпечаток — разобранные поля, не сырые байты: порядок ключей JSON и отсутствующий attachment_ids равны пустому списку. + msg, err := a.store.AddMessage(r.Context(), id, user.ID, body.Text, ids, idemKey, messageFingerprint(body.Text, ids)) + if a.writeStoreErr(w, r, err) { + return + } + writeJSON(w, http.StatusCreated, asMessage(msg)) +} + +func (a *API) listMessages(w http.ResponseWriter, r *http.Request, rawID string) { + user, ok := a.authenticate(w, r) + if !ok { + return + } + id, ok := a.chatID(w, r, rawID) + if !ok { + return + } + if _, err := a.store.ChatFor(r.Context(), id, user.ID); a.writeStoreErr(w, r, err) { + return + } + limit, cur, ok := a.pageQuery(w, r, "msg:"+id) + if !ok { + return + } + items, hasMore, err := a.store.ListMessages(r.Context(), id, user.ID, cur, limit) + if a.writeStoreErr(w, r, err) { + return + } + out := make([]messageJSON, 0, len(items)) + for _, m := range items { + out = append(out, asMessage(m)) + } + var next *string + if hasMore && len(items) > 0 { + last := items[len(items)-1] + encoded := encodeCursor(cursorPayload{Scope: "msg:" + id, ID: last.ID, At: formatTime(last.CreatedAt)}) + next = &encoded + } + writeJSON(w, http.StatusOK, messagePage{Items: out, NextCursor: next}) +} + +func (a *API) chatID(w http.ResponseWriter, r *http.Request, raw string) (string, bool) { + id, ok := canonicalUUID(raw) + if !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный идентификатор", 0) + return "", false + } + return id, true +} + +func (a *API) writeStoreErr(w http.ResponseWriter, r *http.Request, err error) bool { + if err == nil { + return false + } + switch { + case errors.Is(err, store.ErrUnavailable), errors.Is(err, session.ErrUnavailable): + a.fail(w, r, http.StatusServiceUnavailable, "unavailable", "Сервис временно недоступен", 1) + case errors.Is(err, store.ErrIdempotencyMismatch): + a.fail(w, r, http.StatusConflict, "conflict", "Этот ключ уже использован для другого сообщения", 0) + case errors.Is(err, store.ErrConflict): + a.fail(w, r, http.StatusConflict, "conflict", "Имя пользователя уже занято", 0) + case errors.Is(err, store.ErrNotFound): + a.fail(w, r, http.StatusNotFound, "not_found", "Объект не найден", 0) + case errors.Is(err, store.ErrNotReady): + a.fail(w, r, http.StatusConflict, "conflict", "Вложение нельзя прикрепить", 0) + case errors.Is(err, store.ErrInvalid): + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректные данные", 0) + case errors.Is(err, store.ErrBadCursor): + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный cursor", 0) + default: + a.log.Error("store", "request_id", requestIDFrom(r)) + a.fail(w, r, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", 0) + } + return true +} + +func splitChatPath(path string) (id, kind string, ok bool) { + const prefix = "/api/v1/chats/" + if !strings.HasPrefix(path, prefix) { + return "", "", false + } + rest := strings.TrimPrefix(path, prefix) + if rest == "" { + return "", "", false + } + for _, suffix := range []string{"/messages", "/attachments"} { + if strings.HasSuffix(rest, suffix) { + id = strings.TrimSuffix(rest, suffix) + if id == "" || strings.Contains(id, "/") { + return "", "", false + } + return id, strings.TrimPrefix(suffix, "/"), true + } + } + if strings.Contains(rest, "/") { + return "", "", false + } + return rest, "", true +} diff --git a/internal/api/http.go b/internal/api/http.go new file mode 100644 index 00000000..b535cb42 --- /dev/null +++ b/internal/api/http.go @@ -0,0 +1,338 @@ +package api + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "reflect" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" + + "messenger/internal/store" +) + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) +var uuidRe = regexp.MustCompile(`(?i)^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +var limitRe = regexp.MustCompile(`^[0-9]{1,3}$`) +var requestIDRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,64}$`) +var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +type errorEnvelope struct { + Error errorObject `json:"error"` +} + +type errorObject struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type userJSON struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt string `json:"created_at"` +} + +type chatJSON struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt string `json:"created_at"` +} + +type messageJSON struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt string `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type chatPage struct { + Items []chatJSON `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type messagePage struct { + Items []messageJSON `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type cursorPayload struct { + Scope string `json:"scope"` + ID string `json:"id"` + At string `json:"at"` +} + +type statusWriter struct { + http.ResponseWriter + status int + wrote bool +} + +func (s *statusWriter) WriteHeader(code int) { + if s.wrote { + return + } + s.status = code + s.wrote = true + s.ResponseWriter.WriteHeader(code) +} + +func (s *statusWriter) Write(b []byte) (int, error) { + if !s.wrote { + s.WriteHeader(http.StatusOK) + } + return s.ResponseWriter.Write(b) +} + +func (a *API) fail(w http.ResponseWriter, r *http.Request, status int, code, message string, retryAfter int) { + if retryAfter > 0 { + w.Header().Set("Retry-After", strconv.Itoa(retryAfter)) + } + writeJSON(w, status, errorEnvelope{Error: errorObject{ + Code: code, + Message: message, + RequestID: requestIDFrom(r), + }}) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func (a *API) readJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + media := "" + if ct := r.Header.Get("Content-Type"); ct != "" { + media = strings.TrimSpace(strings.Split(ct, ";")[0]) + } + if !strings.EqualFold(media, "application/json") { + a.fail(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "Нужен Content-Type application/json", 0) + return false + } + body, err := io.ReadAll(r.Body) + if err != nil || !utf8.Valid(body) || !decodeExactJSON(body, dst) { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный JSON", 0) + return false + } + return true +} + +// decodeExactJSON принимает один JSON-объект: точные имена полей, без хвоста и без замены битого UTF-8. +func decodeExactJSON(body []byte, dst any) bool { + dec := json.NewDecoder(bytes.NewReader(body)) + var fields map[string]json.RawMessage + if err := dec.Decode(&fields); err != nil || fields == nil { + return false + } + var trailing json.RawMessage + if err := dec.Decode(&trailing); err != io.EOF { + return false + } + allowed, ok := jsonFieldNames(dst) + if !ok { + return false + } + for name := range fields { + if _, ok := allowed[name]; !ok { + return false + } + } + return json.Unmarshal(body, dst) == nil +} + +func jsonFieldNames(dst any) (map[string]struct{}, bool) { + t := reflect.TypeOf(dst) + if t == nil || t.Kind() != reflect.Pointer { + return nil, false + } + t = t.Elem() + if t.Kind() != reflect.Struct { + return nil, false + } + names := make(map[string]struct{}, t.NumField()) + for i := 0; i < t.NumField(); i++ { + name, _, _ := strings.Cut(t.Field(i).Tag.Get("json"), ",") + if name == "" || name == "-" { + return nil, false + } + names[name] = struct{}{} + } + return names, true +} + +func validPassword(s string) bool { + if !utf8.ValidString(s) { + return false + } + n := utf8.RuneCountInString(s) + return n >= 12 && n <= 128 && strings.TrimSpace(s) != "" +} + +func (a *API) pageQuery(w http.ResponseWriter, r *http.Request, scope string) (int, *store.Cursor, bool) { + q := r.URL.Query() + limit := 50 + if q.Has("limit") { + raw := q.Get("limit") + n, err := strconv.Atoi(raw) + if err != nil || !limitRe.MatchString(raw) || n < 1 || n > 100 { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный limit", 0) + return 0, nil, false + } + limit = n + } + if !q.Has("cursor") { + return limit, nil, true + } + raw := q.Get("cursor") + if raw == "" || len(raw) > 512 { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный cursor", 0) + return 0, nil, false + } + payload, err := decodeCursor(raw) + if err != nil || payload.Scope != scope || payload.ID == "" || payload.At == "" { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный cursor", 0) + return 0, nil, false + } + at, err := time.Parse("2006-01-02T15:04:05.000000Z", payload.At) + if err != nil { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный cursor", 0) + return 0, nil, false + } + id, ok := canonicalUUID(payload.ID) + if !ok { + a.fail(w, r, http.StatusBadRequest, "invalid_request", "Некорректный cursor", 0) + return 0, nil, false + } + return limit, &store.Cursor{ID: id, At: at}, true +} + +func asUser(u store.User) userJSON { + return userJSON{ + ID: u.ID, + Username: u.Username, + DisplayName: u.DisplayName, + CreatedAt: formatTime(u.CreatedAt), + } +} + +func asChat(c store.Chat) chatJSON { + ids := c.MemberIDs + if ids == nil { + ids = []string{} + } + return chatJSON{ + ID: c.ID, + Title: c.Title, + MemberIDs: ids, + CreatedAt: formatTime(c.CreatedAt), + } +} + +func asMessage(m store.Message) messageJSON { + ids := m.AttachmentIDs + if ids == nil { + ids = []string{} + } + return messageJSON{ + ID: m.ID, + ChatID: m.ChatID, + SenderID: m.SenderID, + Text: m.Text, + CreatedAt: formatTime(m.CreatedAt), + AttachmentIDs: ids, + } +} + +func messageFingerprint(text string, ids []string) string { + if len(ids) == 0 { + return text + "\n" + } + return text + "\n" + strings.Join(ids, ",") +} + +func validMessageText(text string, attachments int) bool { + if utf8.RuneCountInString(text) > 4000 { + return false + } + if strings.TrimSpace(text) == "" { + return text == "" && attachments > 0 + } + return true +} + +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func canonicalUUID(s string) (string, bool) { + if !uuidRe.MatchString(s) { + return "", false + } + return strings.ToLower(s), true +} + +func validText(s string, min, max int) bool { + if strings.TrimSpace(s) == "" { + return false + } + n := utf8.RuneCountInString(s) + return n >= min && n <= max +} + +func requestID(r *http.Request) string { + incoming := strings.TrimSpace(r.Header.Get("X-Request-Id")) + if requestIDRe.MatchString(incoming) { + return incoming + } + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return strconv.FormatInt(time.Now().UnixNano(), 36) + } + return hex.EncodeToString(b[:]) +} + +func encodeCursor(p cursorPayload) string { + raw, err := json.Marshal(p) + if err != nil { + return "" + } + return base64.RawURLEncoding.EncodeToString(raw) +} + +func decodeCursor(s string) (cursorPayload, error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + b, err = base64.URLEncoding.DecodeString(s) + if err != nil { + return cursorPayload{}, err + } + } + dec := json.NewDecoder(bytes.NewReader(b)) + dec.DisallowUnknownFields() + var p cursorPayload + if err := dec.Decode(&p); err != nil { + return cursorPayload{}, err + } + var trailing json.RawMessage + if err := dec.Decode(&trailing); err != io.EOF { + if err == nil { + err = io.ErrUnexpectedEOF + } + return cursorPayload{}, err + } + return p, nil +} diff --git a/internal/api/server.go b/internal/api/server.go new file mode 100644 index 00000000..a15b9e39 --- /dev/null +++ b/internal/api/server.go @@ -0,0 +1,130 @@ +package api + +import ( + "context" + "log/slog" + "net/http" + "sync/atomic" + "time" + + "messenger/internal/blob" + "messenger/internal/password" + "messenger/internal/session" + "messenger/internal/store" +) + +type ctxKey struct{} + +type API struct { + store store.Repository + sessions session.Store + hasher password.Params + dummyHash string + sessionTTL time.Duration + instanceID string + log *slog.Logger + shutting atomic.Bool + blobs blob.Store + maxUpload int64 + maxPixels int +} + +// New связывает HTTP с хранилищем и сессиями. +// dummyHash нужен, чтобы несуществующий username тратил столько же KDF, сколько существующий. +func New(st store.Repository, sessions session.Store, hasher password.Params, sessionTTL time.Duration, instanceID string, log *slog.Logger) *API { + if log == nil { + log = slog.Default() + } + if sessionTTL <= 0 { + sessionTTL = time.Hour + } + if instanceID == "" { + instanceID = "api" + } + dummy, err := hasher.Hash("dummy-password-not-a-login") + if err != nil { + // Параметры KDF проверяются на старте. Пустая строка заставит Verify сразу вернуть false. + dummy = "" + } + return &API{ + store: st, + sessions: sessions, + hasher: hasher, + dummyHash: dummy, + sessionTTL: sessionTTL, + instanceID: instanceID, + log: log, + } +} + +// EnableAttachments подключает приватный bucket и жёсткие лимиты загрузки. +// maxUpload не может превысить 10 МиБ контракта. +func (a *API) EnableAttachments(objects blob.Store, maxUpload int64, maxPixels int) { + const contractMax = 10 * 1024 * 1024 + if maxUpload <= 0 || maxUpload > contractMax { + maxUpload = contractMax + } + if maxPixels <= 0 || maxPixels > 40_000_000 { + maxPixels = 40_000_000 + } + a.blobs = objects + a.maxUpload = maxUpload + a.maxPixels = maxPixels +} + +// BeginShutdown переводит /health/ready в 503 и говорит новым бизнес-запросам +// отказать до закрытия слушателя. Уже принятые обработчики дорабатывают. +func (a *API) BeginShutdown() { + a.shutting.Store(true) + a.log.Info("shutdown started") +} + +// Точка входа +func (a *API) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} + start := time.Now() + id := requestID(r) + sw.Header().Set("X-Request-Id", id) + sw.Header().Set("X-Instance-Id", a.instanceID) + r = r.WithContext(context.WithValue(r.Context(), ctxKey{}, id)) + + defer func() { + if p := recover(); p != nil { + a.log.Error("panic", "request_id", id) + if !sw.wrote { + a.fail(sw, r, http.StatusInternalServerError, "internal_error", "Внутренняя ошибка", 0) + } + } + a.log.Info("request", + "request_id", id, + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", time.Since(start).Milliseconds(), + "bytes_in", r.ContentLength, + ) + }() + + switch { + case r.Method == http.MethodGet && r.URL.Path == "/health/live": + a.live(sw, r) + return + case r.Method == http.MethodGet && r.URL.Path == "/health/ready": + a.ready(sw, r) + return + } + + if a.shutting.Load() { + a.fail(sw, r, http.StatusServiceUnavailable, "unavailable", "Сервер останавливается", 1) + return + } + a.route(sw, r) +} + +func requestIDFrom(r *http.Request) string { + id, _ := r.Context().Value(ctxKey{}).(string) + if id == "" { + return "unknown" + } + return id +} diff --git a/internal/api/server_test.go b/internal/api/server_test.go new file mode 100644 index 00000000..ccb94128 --- /dev/null +++ b/internal/api/server_test.go @@ -0,0 +1,842 @@ +package api + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "image" + "image/png" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "messenger/internal/blob" + "messenger/internal/password" + "messenger/internal/session" + "messenger/internal/store" +) + +const testPassword = "correct-horse-1" + +type labServer struct { + URL string + tokens map[string]string + api *API + store store.Repository + sess session.Store +} + +func newServer(t *testing.T) *labServer { + return newServerTTL(t, time.Hour) +} + +func newServerTTL(t *testing.T, ttl time.Duration) *labServer { + t.Helper() + st := store.New() + sess := session.NewMemory() + application := New(st, sess, password.Params{Time: 1, Memory: 8, Threads: 1}, ttl, "test-instance", slog.New(slog.NewJSONHandler(io.Discard, nil))) + srv := httptest.NewServer(application) + t.Cleanup(srv.Close) + return &labServer{URL: srv.URL, tokens: map[string]string{}, api: application, store: st, sess: sess} +} + +func (e *labServer) bearer(user string) string { + if user == "" { + return "" + } + if tok, ok := e.tokens[user]; ok { + return tok + } + return "unknown-session-token" +} + +var keySeq atomic.Uint64 + +func nextIdemKey() string { + return "k" + strconv.FormatUint(keySeq.Add(1), 36) +} + +func do(t *testing.T, env *labServer, method, url, user string, body any, raw []byte, contentType string, extra ...map[string]string) (int, http.Header, []byte) { + t.Helper() + var rdr io.Reader + if raw != nil { + rdr = bytes.NewReader(raw) + } else if body != nil { + b, err := json.Marshal(body) + if err != nil { + t.Fatal(err) + } + rdr = bytes.NewReader(b) + } + req, err := http.NewRequest(method, url, rdr) + if err != nil { + t.Fatal(err) + } + if body != nil || raw != nil { + if contentType == "" { + contentType = "application/json" + } + req.Header.Set("Content-Type", contentType) + } + if tok := env.bearer(user); tok != "" { + req.Header.Set("Authorization", "Bearer "+tok) + } + if len(extra) > 0 { + for k, v := range extra[0] { + req.Header.Set(k, v) + } + } + if method == http.MethodPost && strings.Contains(url, "/messages") && req.Header.Get("Idempotency-Key") == "" { + req.Header.Set("Idempotency-Key", nextIdemKey()) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if resp.Header.Get("X-Request-Id") == "" { + t.Fatalf("%s %s: missing X-Request-Id", method, url) + } + if resp.Header.Get("X-Instance-Id") != "test-instance" { + t.Fatalf("%s %s: X-Instance-Id = %q", method, url, resp.Header.Get("X-Instance-Id")) + } + if resp.StatusCode >= 400 { + var env errorEnvelope + if err := json.Unmarshal(data, &env); err != nil { + t.Fatalf("error json: %v body %s", err, data) + } + if env.Error.RequestID != resp.Header.Get("X-Request-Id") || env.Error.Code == "" || env.Error.Message == "" { + t.Fatalf("bad error envelope: %+v", env) + } + } + return resp.StatusCode, resp.Header, data +} + +func createUser(t *testing.T, env *labServer, username, display string) userJSON { + t.Helper() + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/users", "", map[string]string{ + "username": username, "display_name": display, "password": testPassword, + }, nil, "") + if status != http.StatusCreated { + t.Fatalf("create user %s: %d %s", username, status, body) + } + if bytes.Contains(body, []byte("password")) || bytes.Contains(body, []byte("argon2")) { + t.Fatalf("user response leaked a secret: %s", body) + } + var u userJSON + if err := json.Unmarshal(body, &u); err != nil { + t.Fatal(err) + } + req, err := http.NewRequest(http.MethodPost, env.URL+"/api/v1/auth/sessions", nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth(username, testPassword) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("login %s: %d %s", username, resp.StatusCode, raw) + } + var sess struct { + Token string `json:"token"` + } + if err := json.Unmarshal(raw, &sess); err != nil || len(sess.Token) < 32 { + t.Fatalf("session: %s", raw) + } + env.tokens[u.ID] = sess.Token + return u +} + +func createChat(t *testing.T, env *labServer, caller, title string, members ...string) chatJSON { + t.Helper() + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/chats", caller, map[string]any{ + "title": title, "member_ids": members, + }, nil, "") + if status != http.StatusCreated { + t.Fatalf("create chat: %d %s", status, body) + } + var c chatJSON + if err := json.Unmarshal(body, &c); err != nil { + t.Fatal(err) + } + return c +} + +func TestInvalidJSONExtraFieldAndTypes(t *testing.T) { + env := newServer(t) + cases := [][]byte{ + []byte(`{`), + []byte(`{"username":"abc","display_name":"Алиса","extra":1}`), + []byte(`{"username":1,"display_name":"Алиса"}`), + []byte(`null`), + } + for _, raw := range cases { + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/users", "", nil, raw, "application/json") + if status != http.StatusBadRequest || !bytes.Contains(body, []byte(`"invalid_request"`)) { + t.Fatalf("body %s: %d %s", raw, status, body) + } + } + status, _, _ := do(t, env, http.MethodPost, env.URL+"/api/v1/users", "", nil, []byte(`{"username":"abc","display_name":"Алиса"}`), "text/plain") + if status != http.StatusUnsupportedMediaType { + t.Fatalf("content type: %d", status) + } +} + +func TestStrictJSONNamesUTF8AndCursorTail(t *testing.T) { + env := newServer(t) + bad := [][]byte{ + []byte(`{"username":"bad_json","display_name":"Я"}]`), + []byte(`{"username":"bad_json","display_name":"Я"}}`), + []byte(`{"Username":"bad_case","display_name":"Я"}`), + append([]byte(`{"username":"bad_utf","display_name":"`), 0xff, '"', '}'), + } + for _, raw := range bad { + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/users", "", nil, raw, "application/json") + if status != http.StatusBadRequest { + t.Fatalf("user body %q: %d %s", raw, status, body) + } + } + + alice := createUser(t, env, "alice", "Алиса") + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/chats", alice.ID, nil, []byte(`{"Title":"чат","member_ids":["`+alice.ID+`"]}`), "application/json") + if status != http.StatusBadRequest { + t.Fatalf("Title: %d %s", status, body) + } + chat := createChat(t, env, alice.ID, "чат", alice.ID) + + status, _, body = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, nil, []byte(`{"TEXT":"hi"}`), "application/json") + if status != http.StatusBadRequest { + t.Fatalf("TEXT: %d %s", status, body) + } + rawText := append([]byte(`{"text":"`), 0xff, '"', '}') + status, _, body = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, nil, rawText, "application/json") + if status != http.StatusBadRequest { + t.Fatalf("message utf-8: %d %s", status, body) + } + + status, _, body = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]string{"text": "one"}, nil, "") + if status != http.StatusCreated { + t.Fatalf("seed: %d %s", status, body) + } + status, _, body = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]string{"text": "two"}, nil, "") + if status != http.StatusCreated { + t.Fatalf("seed2: %d %s", status, body) + } + status, _, body = do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+chat.ID+"/messages?limit=1", alice.ID, nil, nil, "") + if status != http.StatusOK { + t.Fatalf("page: %d %s", status, body) + } + var page messagePage + if err := json.Unmarshal(body, &page); err != nil || page.NextCursor == nil { + t.Fatalf("cursor: %v %s", err, body) + } + decoded, err := base64.RawURLEncoding.DecodeString(*page.NextCursor) + if err != nil { + t.Fatal(err) + } + tampered := base64.RawURLEncoding.EncodeToString(append(decoded, []byte("garbage")...)) + status, _, body = do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+chat.ID+"/messages?limit=1&cursor="+url.QueryEscape(tampered), alice.ID, nil, nil, "") + if status != http.StatusBadRequest { + t.Fatalf("tampered cursor: %d %s", status, body) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+chat.ID+"/messages?limit=1&cursor="+url.QueryEscape(*page.NextCursor), alice.ID, nil, nil, "") + if status != http.StatusOK { + t.Fatalf("valid cursor: %d", status) + } +} + +func TestDuplicateUsername(t *testing.T) { + env := newServer(t) + createUser(t, env, "alice", "Алиса") + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/users", "", map[string]string{ + "username": "alice", "display_name": "Другая", "password": testPassword, + }, nil, "") + if status != http.StatusConflict || !bytes.Contains(body, []byte(`"conflict"`)) { + t.Fatalf("duplicate: %d %s", status, body) + } +} + +func TestEmptyMessageForeignChatAndAuth(t *testing.T) { + env := newServer(t) + alice := createUser(t, env, "alice", "Алиса") + bob := createUser(t, env, "bob", "Боб") + eve := createUser(t, env, "eve", "Ева") + chat := createChat(t, env, alice.ID, "Контрактный тест", alice.ID, bob.ID) + + status, _, _ := do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", "", nil, nil, "") + if status != http.StatusUnauthorized { + t.Fatalf("missing header: %d", status) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", "11111111-1111-4111-8111-111111111111", nil, nil, "") + if status != http.StatusUnauthorized { + t.Fatalf("unknown user: %d", status) + } + meStatus, _, meBody := do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", alice.ID, nil, nil, "") + if meStatus != http.StatusOK || !bytes.Contains(meBody, []byte(alice.ID)) { + t.Fatalf("me: %d %s", meStatus, meBody) + } + + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+chat.ID, eve.ID, nil, nil, "") + if status != http.StatusNotFound { + t.Fatalf("foreign chat: %d", status) + } + status, _, _ = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", eve.ID, map[string]string{"text": "чужое"}, nil, "") + if status != http.StatusNotFound { + t.Fatalf("foreign post: %d", status) + } + for _, text := range []string{"", " ", "\n\t"} { + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]string{"text": text}, nil, "") + if status != http.StatusBadRequest { + t.Fatalf("empty %q: %d %s", text, status, body) + } + } + status, _, _ = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]string{"text": "привет", "sender_id": eve.ID}, nil, "") + if status != http.StatusBadRequest { + t.Fatalf("sender_id must be rejected: %d", status) + } + + text := "Привет, мир 👋" + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]string{"text": text}, nil, "") + if status != http.StatusCreated { + t.Fatalf("unicode: %d %s", status, body) + } + var msg messageJSON + if err := json.Unmarshal(body, &msg); err != nil { + t.Fatal(err) + } + if msg.Text != text || msg.SenderID != alice.ID || msg.ChatID != chat.ID { + t.Fatalf("message mutated: %+v", msg) + } + if !strings.HasSuffix(msg.CreatedAt, "Z") { + t.Fatalf("timestamp: %s", msg.CreatedAt) + } +} + +func TestPaginationSeveralPages(t *testing.T) { + env := newServer(t) + alice := createUser(t, env, "alice", "Алиса") + bob := createUser(t, env, "bob", "Боб") + chat := createChat(t, env, alice.ID, "страницы", alice.ID, bob.ID) + other := createChat(t, env, alice.ID, "другой", alice.ID) + const n = 25 + ids := make([]string, 0, n) + for i := 0; i < n; i++ { + sender := alice.ID + if i%2 == 1 { + sender = bob.ID + } + status, _, body := do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", sender, map[string]string{ + "text": "сообщение " + strings.Repeat("я", i%3) + string(rune('a'+i%26)), + }, nil, "") + if status != http.StatusCreated { + t.Fatalf("post %d: %d %s", i, status, body) + } + var msg messageJSON + if err := json.Unmarshal(body, &msg); err != nil { + t.Fatal(err) + } + ids = append(ids, msg.ID) + } + var got []string + cursor := "" + pages := 0 + for { + pages++ + if pages > 10 { + t.Fatal("pagination did not finish") + } + url := env.URL + "/api/v1/chats/" + chat.ID + "/messages?limit=10" + if cursor != "" { + url += "&cursor=" + cursor + } + status, _, body := do(t, env, http.MethodGet, url, bob.ID, nil, nil, "") + if status != http.StatusOK { + t.Fatalf("page: %d %s", status, body) + } + var page messagePage + if err := json.Unmarshal(body, &page); err != nil { + t.Fatal(err) + } + if len(page.Items) == 0 || len(page.Items) > 10 { + t.Fatalf("page size %d", len(page.Items)) + } + for _, item := range page.Items { + got = append(got, item.ID) + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if pages < 3 { + t.Fatalf("expected several pages, got %d", pages) + } + if strings.Join(got, ",") != strings.Join(ids, ",") { + t.Fatalf("order/duplication mismatch\nwant %v\ngot %v", ids, got) + } + status, _, _ := do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+other.ID+"/messages?limit=10&cursor="+cursor, alice.ID, nil, nil, "") + if status != http.StatusBadRequest { + t.Fatalf("foreign cursor: %d", status) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/chats/"+chat.ID+"/messages?limit=0", alice.ID, nil, nil, "") + if status != http.StatusBadRequest { + t.Fatalf("limit 0: %d", status) + } +} + +func TestConcurrentHundredMessages(t *testing.T) { + env := newServer(t) + users := make([]userJSON, 10) + ids := make([]string, 10) + for i := 0; i < 10; i++ { + users[i] = createUser(t, env, "u"+string(rune('a'+i))+"x", "Участник") + ids[i] = users[i].ID + } + chat := createChat(t, env, ids[0], "гонка", ids...) + var wg sync.WaitGroup + errCh := make(chan error, 100) + for i := 0; i < 10; i++ { + wg.Add(1) + go func(user string) { + defer wg.Done() + for j := 0; j < 10; j++ { + req, err := http.NewRequest(http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", strings.NewReader(`{"text":"n"}`)) + if err != nil { + errCh <- err + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+env.tokens[user]) + req.Header.Set("Idempotency-Key", user+"-"+strconv.Itoa(j)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + errCh <- err + return + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + errCh <- &statusErr{resp.StatusCode, body} + } + } + }(ids[i]) + } + wg.Wait() + close(errCh) + for err := range errCh { + t.Fatal(err) + } + seen := map[string]struct{}{} + cursor := "" + for { + url := env.URL + "/api/v1/chats/" + chat.ID + "/messages?limit=17" + if cursor != "" { + url += "&cursor=" + cursor + } + status, _, body := do(t, env, http.MethodGet, url, ids[3], nil, nil, "") + if status != http.StatusOK { + t.Fatalf("list: %d %s", status, body) + } + var page messagePage + if err := json.Unmarshal(body, &page); err != nil { + t.Fatal(err) + } + for _, item := range page.Items { + if _, ok := seen[item.ID]; ok { + t.Fatalf("duplicate %s", item.ID) + } + seen[item.ID] = struct{}{} + } + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + } + if len(seen) != 100 { + t.Fatalf("unique messages = %d, want 100", len(seen)) + } +} + +func TestSessionWrongPasswordLogoutAndIgnoredUserHeader(t *testing.T) { + env := newServer(t) + alice := createUser(t, env, "alice", "Алиса") + eve := createUser(t, env, "eve", "Ева") + + req, err := http.NewRequest(http.MethodPost, env.URL+"/api/v1/auth/sessions", nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth("alice", "wrong-password-1") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + body, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized || !strings.HasPrefix(resp.Header.Get("WWW-Authenticate"), "Basic ") { + t.Fatalf("wrong password: %d %s %q", resp.StatusCode, body, resp.Header.Get("WWW-Authenticate")) + } + + status, hdr, _ := do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", "", nil, nil, "") + if status != http.StatusUnauthorized || !strings.HasPrefix(hdr.Get("WWW-Authenticate"), "Bearer ") { + t.Fatalf("missing bearer: %d %q", status, hdr.Get("WWW-Authenticate")) + } + bare, err := http.NewRequest(http.MethodGet, env.URL+"/api/v1/users/me", nil) + if err != nil { + t.Fatal(err) + } + bare.Header.Set("X-User-Id", alice.ID) + resp, err = http.DefaultClient.Do(bare) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("X-User-Id alone: %d", resp.StatusCode) + } + over, err := http.NewRequest(http.MethodGet, env.URL+"/api/v1/users/me", nil) + if err != nil { + t.Fatal(err) + } + over.Header.Set("Authorization", "Bearer "+env.tokens[alice.ID]) + over.Header.Set("X-User-Id", eve.ID) + resp, err = http.DefaultClient.Do(over) + if err != nil { + t.Fatal(err) + } + raw, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || !bytes.Contains(raw, []byte(alice.ID)) || bytes.Contains(raw, []byte(eve.ID)) { + t.Fatalf("X-User-Id overrode bearer: %d %s", resp.StatusCode, raw) + } + + basic, err := http.NewRequest(http.MethodGet, env.URL+"/api/v1/users/me", nil) + if err != nil { + t.Fatal(err) + } + basic.SetBasicAuth("alice", testPassword) + resp, err = http.DefaultClient.Do(basic) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized || !strings.HasPrefix(resp.Header.Get("WWW-Authenticate"), "Bearer ") { + t.Fatalf("basic on business endpoint: %d", resp.StatusCode) + } + + status, _, body = do(t, env, http.MethodDelete, env.URL+"/api/v1/auth/sessions/current", alice.ID, nil, nil, "") + if status != http.StatusNoContent || len(body) != 0 { + t.Fatalf("logout: %d %q", status, body) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", alice.ID, nil, nil, "") + if status != http.StatusUnauthorized { + t.Fatalf("revoked token: %d", status) + } + status, _, _ = do(t, env, http.MethodDelete, env.URL+"/api/v1/auth/sessions/current", alice.ID, nil, nil, "") + if status != http.StatusUnauthorized { + t.Fatalf("second logout: %d", status) + } +} + +func TestSessionExpires(t *testing.T) { + env := newServerTTL(t, 2*time.Second) + alice := createUser(t, env, "alice", "Алиса") + time.Sleep(2100 * time.Millisecond) + status, _, _ := do(t, env, http.MethodGet, env.URL+"/api/v1/users/me", alice.ID, nil, nil, "") + if status != http.StatusUnauthorized { + t.Fatalf("expired session: %d", status) + } +} + +type downSessions struct{ session.Store } + +func (downSessions) Lookup(context.Context, string) (string, time.Time, error) { + return "", time.Time{}, session.ErrUnavailable +} +func (downSessions) Ping(context.Context) error { return session.ErrUnavailable } + +type downStore struct{ store.Repository } + +func (downStore) Ping(context.Context) error { return store.ErrUnavailable } +func (downStore) User(context.Context, string) (store.User, error) { + return store.User{}, store.ErrUnavailable +} + +func TestDependencyDownDoesNotTrustAnonymous(t *testing.T) { + env := newServer(t) + alice := createUser(t, env, "alice", "Алиса") + broken := New(downStore{env.store}, downSessions{env.sess}, password.Params{Time: 1, Memory: 8, Threads: 1}, time.Hour, "test-instance", slog.New(slog.NewJSONHandler(io.Discard, nil))) + srv := httptest.NewServer(broken) + defer srv.Close() + + live, err := http.Get(srv.URL + "/health/live") + if err != nil { + t.Fatal(err) + } + live.Body.Close() + if live.StatusCode != http.StatusOK { + t.Fatalf("live: %d", live.StatusCode) + } + ready, err := http.Get(srv.URL + "/health/ready") + if err != nil { + t.Fatal(err) + } + readyBody, _ := io.ReadAll(ready.Body) + ready.Body.Close() + if ready.StatusCode != http.StatusServiceUnavailable || !bytes.Contains(readyBody, []byte(`"unavailable"`)) { + t.Fatalf("ready: %d %s", ready.StatusCode, readyBody) + } + + anon, err := http.Get(srv.URL + "/api/v1/users/me") + if err != nil { + t.Fatal(err) + } + anon.Body.Close() + if anon.StatusCode != http.StatusUnauthorized { + t.Fatalf("anonymous while down: %d", anon.StatusCode) + } + authed, err := http.NewRequest(http.MethodGet, srv.URL+"/api/v1/users/me", nil) + if err != nil { + t.Fatal(err) + } + authed.Header.Set("Authorization", "Bearer "+env.tokens[alice.ID]) + resp, err := http.DefaultClient.Do(authed) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("bearer while session store is down: %d", resp.StatusCode) + } +} + +func TestIdempotencyReplayConflictAndRace(t *testing.T) { + env := newServer(t) + alice := createUser(t, env, "alice", "Алиса") + bob := createUser(t, env, "bob", "Боб") + chat := createChat(t, env, alice.ID, "ключ", alice.ID, bob.ID) + path := env.URL + "/api/v1/chats/" + chat.ID + "/messages" + key := "same-key-1" + + status, _, body := do(t, env, http.MethodPost, path, alice.ID, map[string]string{"text": "одно"}, nil, "", map[string]string{"Idempotency-Key": key}) + if status != http.StatusCreated { + t.Fatalf("first: %d %s", status, body) + } + var first messageJSON + if err := json.Unmarshal(body, &first); err != nil { + t.Fatal(err) + } + status, _, body = do(t, env, http.MethodPost, path, alice.ID, nil, []byte(`{"text":"одно"}`), "application/json", map[string]string{"Idempotency-Key": key}) + if status != http.StatusCreated { + t.Fatalf("replay: %d %s", status, body) + } + var replay messageJSON + if err := json.Unmarshal(body, &replay); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(replay, first) { + t.Fatalf("replay changed message: %+v vs %+v", replay, first) + } + status, _, _ = do(t, env, http.MethodPost, path, alice.ID, map[string]string{"text": "другое"}, nil, "", map[string]string{"Idempotency-Key": key}) + if status != http.StatusConflict { + t.Fatalf("mismatch: %d", status) + } + req, err := http.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"без ключа"}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+env.tokens[alice.ID]) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("missing key: %d", resp.StatusCode) + } + + status, _, body = do(t, env, http.MethodPost, path, bob.ID, map[string]string{"text": "одно"}, nil, "", map[string]string{"Idempotency-Key": key}) + if status != http.StatusCreated { + t.Fatalf("other user: %d %s", status, body) + } + var other messageJSON + if err := json.Unmarshal(body, &other); err != nil { + t.Fatal(err) + } + if other.ID == first.ID { + t.Fatal("same key for another user reused the message") + } + + raceKey := "race-key" + var wg sync.WaitGroup + ids := make(chan string, 16) + errs := make(chan error, 16) + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req, err := http.NewRequest(http.MethodPost, path, strings.NewReader(`{"text":"гонка"}`)) + if err != nil { + errs <- err + return + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+env.tokens[alice.ID]) + req.Header.Set("Idempotency-Key", raceKey) + resp, err := http.DefaultClient.Do(req) + if err != nil { + errs <- err + return + } + defer resp.Body.Close() + raw, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusCreated { + errs <- &statusErr{resp.StatusCode, raw} + return + } + var msg messageJSON + if err := json.Unmarshal(raw, &msg); err != nil { + errs <- err + return + } + ids <- msg.ID + }() + } + wg.Wait() + close(ids) + close(errs) + for err := range errs { + t.Fatal(err) + } + seen := map[string]struct{}{} + for id := range ids { + seen[id] = struct{}{} + } + if len(seen) != 1 { + t.Fatalf("concurrent same key created %d messages", len(seen)) + } +} + +func pngBytes(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestAttachmentRoundTripAndRejections(t *testing.T) { + env := newServer(t) + env.api.EnableAttachments(blob.NewMemory(), 10*1024*1024, 40_000_000) + alice := createUser(t, env, "alice", "Алиса") + bob := createUser(t, env, "bob", "Боб") + eve := createUser(t, env, "eve", "Ева") + chat := createChat(t, env, alice.ID, "файлы", alice.ID, bob.ID) + other := createChat(t, env, alice.ID, "другой", alice.ID) + body := pngBytes(t, 4, 3) + + status, _, raw := do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/attachments?filename=a/b.png", alice.ID, nil, body, "image/png") + if status != http.StatusBadRequest { + t.Fatalf("slash filename: %d %s", status, raw) + } + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/attachments?filename=note.png", alice.ID, nil, []byte("hello"), "image/png") + if status != http.StatusUnsupportedMediaType { + t.Fatalf("text as png: %d %s", status, raw) + } + jpegHeader := []byte{0xff, 0xd8, 0xff, 0xd9} + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/attachments?filename=x.png", alice.ID, nil, jpegHeader, "image/png") + if status != http.StatusUnsupportedMediaType { + t.Fatalf("mime swap: %d %s", status, raw) + } + + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/attachments?filename=pic.png", alice.ID, nil, body, "image/png") + if status != http.StatusCreated { + t.Fatalf("upload: %d %s", status, raw) + } + var att attachmentJSON + if err := json.Unmarshal(raw, &att); err != nil { + t.Fatal(err) + } + if att.Status != "ready" || att.ErrorCode != nil || att.ChatID != chat.ID || att.OwnerID != alice.ID || len(att.Variants) != 0 { + t.Fatalf("metadata: %+v", att) + } + if bytes.Contains(raw, []byte("object_key")) || bytes.Contains(raw, []byte("\"o/")) { + t.Fatalf("key leaked: %s", raw) + } + status, hdr, got := do(t, env, http.MethodGet, env.URL+"/api/v1/attachments/"+att.ID+"/content", bob.ID, nil, nil, "") + if status != http.StatusOK || hdr.Get("Content-Type") != "image/png" || !bytes.Equal(got, body) { + t.Fatalf("download: %d %s %d", status, hdr.Get("Content-Type"), len(got)) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/attachments/"+att.ID, eve.ID, nil, nil, "") + if status != http.StatusNotFound { + t.Fatalf("outsider meta: %d", status) + } + status, _, _ = do(t, env, http.MethodGet, env.URL+"/api/v1/attachments/"+att.ID+"/content", eve.ID, nil, nil, "") + if status != http.StatusNotFound { + t.Fatalf("outsider bytes: %d", status) + } + + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+other.ID+"/attachments?filename=other.png", alice.ID, nil, body, "image/png") + if status != http.StatusCreated { + t.Fatalf("other upload: %d %s", status, raw) + } + var foreign attachmentJSON + if err := json.Unmarshal(raw, &foreign); err != nil { + t.Fatal(err) + } + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]any{ + "text": "", "attachment_ids": []string{foreign.ID}, + }, nil, "") + if status != http.StatusNotFound { + t.Fatalf("foreign attachment: %d %s", status, raw) + } + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/messages", alice.ID, map[string]any{ + "text": "", "attachment_ids": []string{att.ID}, + }, nil, "") + if status != http.StatusCreated { + t.Fatalf("attach: %d %s", status, raw) + } + var msg messageJSON + if err := json.Unmarshal(raw, &msg); err != nil { + t.Fatal(err) + } + if len(msg.AttachmentIDs) != 1 || msg.AttachmentIDs[0] != att.ID || msg.Text != "" { + t.Fatalf("message: %+v", msg) + } + + env.api.EnableAttachments(blob.NewMemory(), 8, 40_000_000) + status, _, raw = do(t, env, http.MethodPost, env.URL+"/api/v1/chats/"+chat.ID+"/attachments?filename=big.png", alice.ID, nil, body, "image/png") + if status != http.StatusRequestEntityTooLarge { + t.Fatalf("small cap: %d %s", status, raw) + } +} + +type statusErr struct { + code int + body []byte +} + +func (e *statusErr) Error() string { + return http.StatusText(e.code) + " " + string(e.body) +} diff --git a/internal/blob/blob.go b/internal/blob/blob.go new file mode 100644 index 00000000..407b40ea --- /dev/null +++ b/internal/blob/blob.go @@ -0,0 +1,32 @@ +// Хранилище байтов вложения. Реализация — приватный S3-совместимый bucket. +// Ключ задаёт вызывающий код; имя файла пользователя сюда не попадает. +package blob + +import ( + "context" + "errors" + "io" + "time" +) + +var ( + ErrNotFound = errors.New("blob not found") + ErrUnavailable = errors.New("blob unavailable") +) + +// Object — запись листинга. LastModified нужен сверке, чтобы не удалять свежий объект. +type Object struct { + Key string + Size int64 + Modified time.Time +} + +// Store — операции, которые нужны загрузке, скачиванию и сверке. +type Store interface { + Ping(ctx context.Context) error + Put(ctx context.Context, key, contentType string, body []byte) error + Get(ctx context.Context, key string) (io.ReadCloser, error) + Delete(ctx context.Context, key string) error + // List обходит bucket одним листингом. pages — сколько страниц S3 прочитано. + List(ctx context.Context) (objects []Object, pages int, err error) +} diff --git a/internal/blob/memory.go b/internal/blob/memory.go new file mode 100644 index 00000000..edf08ea2 --- /dev/null +++ b/internal/blob/memory.go @@ -0,0 +1,86 @@ +package blob + +import ( + "bytes" + "context" + "io" + "sync" + "time" +) + +type memObj struct { + body []byte + modified time.Time +} + +// Memory - двойник bucket для тестов без MinIO. +type Memory struct { + mu sync.Mutex + m map[string]memObj +} + +func NewMemory() *Memory { + return &Memory{m: map[string]memObj{}} +} + +func (m *Memory) Ping(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return ErrUnavailable + } + return nil +} + +func (m *Memory) Put(ctx context.Context, key, _ string, body []byte) error { + return m.PutAt(ctx, key, body, time.Now().UTC()) +} + +// PutAt записывает объект с заданным временем. +func (m *Memory) PutAt(ctx context.Context, key string, body []byte, modified time.Time) error { + if err := ctx.Err(); err != nil { + return ErrUnavailable + } + m.mu.Lock() + defer m.mu.Unlock() + m.m[key] = memObj{body: append([]byte(nil), body...), modified: modified.UTC()} + return nil +} + +func (m *Memory) Get(ctx context.Context, key string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, ErrUnavailable + } + m.mu.Lock() + defer m.mu.Unlock() + obj, ok := m.m[key] + if !ok { + return nil, ErrNotFound + } + return io.NopCloser(bytes.NewReader(obj.body)), nil +} + +func (m *Memory) Delete(ctx context.Context, key string) error { + if err := ctx.Err(); err != nil { + return ErrUnavailable + } + m.mu.Lock() + defer m.mu.Unlock() + delete(m.m, key) + return nil +} + +func (m *Memory) List(ctx context.Context) ([]Object, int, error) { + if err := ctx.Err(); err != nil { + return nil, 0, ErrUnavailable + } + m.mu.Lock() + defer m.mu.Unlock() + out := make([]Object, 0, len(m.m)) + for key, obj := range m.m { + out = append(out, Object{Key: key, Size: int64(len(obj.body)), Modified: obj.modified}) + } + pages := 1 + if len(out) == 0 { + pages = 1 + } + return out, pages, nil +} diff --git a/internal/blob/s3.go b/internal/blob/s3.go new file mode 100644 index 00000000..8b83eb2d --- /dev/null +++ b/internal/blob/s3.go @@ -0,0 +1,104 @@ +package blob + +import ( + "bytes" + "context" + "errors" + "io" + "net/url" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// S3 - path-style клиент одного bucket. Учётные данные - отдельный пользователь. +type S3 struct { + client *minio.Client + bucket string +} + +func Open(endpoint, region, bucket, accessKey, secretKey string) (*S3, error) { + u, err := url.Parse(endpoint) + if err != nil || u.Host == "" || bucket == "" || accessKey == "" || secretKey == "" { + return nil, errors.New("s3 config") + } + secure := u.Scheme == "https" + client, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: secure, + Region: region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, err + } + return &S3{client: client, bucket: bucket}, nil +} + +func (s *S3) Ping(ctx context.Context) error { + ok, err := s.client.BucketExists(ctx, s.bucket) + if err != nil || !ok { + return ErrUnavailable + } + return nil +} + +func (s *S3) Put(ctx context.Context, key, contentType string, body []byte) error { + _, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(body), int64(len(body)), minio.PutObjectOptions{ + ContentType: contentType, + }) + if err != nil { + return ErrUnavailable + } + return nil +} + +func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) { + _, err := s.client.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + if isMissing(err) { + return nil, ErrNotFound + } + return nil, ErrUnavailable + } + obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, ErrUnavailable + } + return obj, nil +} + +func (s *S3) Delete(ctx context.Context, key string) error { + err := s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) + if err != nil && !isMissing(err) { + return ErrUnavailable + } + return nil +} + +func (s *S3) List(ctx context.Context) ([]Object, int, error) { + ch := s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Recursive: true}) + out := make([]Object, 0) + for obj := range ch { + if obj.Err != nil { + return nil, 0, ErrUnavailable + } + out = append(out, Object{Key: obj.Key, Size: obj.Size, Modified: obj.LastModified.UTC()}) + } + pages := 1 + if n := len(out); n > 0 { + pages = (n + 999) / 1000 + } + return out, pages, nil +} + +func isMissing(err error) bool { + resp := minio.ToErrorResponse(err) + switch resp.Code { + case "NoSuchKey", "NoSuchBucket", "NotFound": + return true + default: + return strings.Contains(resp.Code, "NoSuch") + } +} diff --git a/internal/bootstrap/s3.go b/internal/bootstrap/s3.go new file mode 100644 index 00000000..03aa0e95 --- /dev/null +++ b/internal/bootstrap/s3.go @@ -0,0 +1,118 @@ +// Одноразовый и повторно безопасный bootstrap bucket. +// Root нужен только здесь. Процесс API ходит отдельным ключом. +package bootstrap + +import ( + "context" + "fmt" + "net/url" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +const policyName = "messenger-app" + +// Bucket создаёт приватный bucket, политику и пользователя API, если их ещё нет. +func Bucket(ctx context.Context, endpoint, region, bucket, rootKey, rootSecret, appKey, appSecret string) error { + u, err := url.Parse(endpoint) + if err != nil || u.Host == "" || bucket == "" || rootKey == "" || rootSecret == "" || appKey == "" || appSecret == "" { + return fmt.Errorf("s3 bootstrap config") + } + if region == "" { + region = "us-east-1" + } + secure := u.Scheme == "https" + root, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(rootKey, rootSecret, ""), + Secure: secure, + Region: region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return err + } + var last error + for attempt := 0; attempt < 30; attempt++ { + exists, err := root.BucketExists(ctx, bucket) + if err == nil { + if !exists { + if err := root.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil { + exists, again := root.BucketExists(ctx, bucket) + if again != nil || !exists { + return err + } + } + } + last = nil + break + } + last = err + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Second): + } + } + if last != nil { + return last + } + + adm, err := madmin.New(u.Host, rootKey, rootSecret, secure) + if err != nil { + return err + } + if _, err := adm.InfoCannedPolicy(ctx, policyName); err != nil { + if err := adm.AddCannedPolicy(ctx, policyName, policyJSON(bucket)); err != nil { + return fmt.Errorf("add policy: %w", err) + } + } + if err := adm.AddUser(ctx, appKey, appSecret); err != nil { + return fmt.Errorf("add user: %w", err) + } + if _, err := adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policyName}, + User: appKey, + }); err != nil { + info, infoErr := adm.GetUserInfo(ctx, appKey) + if infoErr != nil || !strings.Contains(info.PolicyName, policyName) { + return fmt.Errorf("attach policy: %w", err) + } + } + + app, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(appKey, appSecret, ""), + Secure: secure, + Region: region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return err + } + ok, err := app.BucketExists(ctx, bucket) + if err != nil || !ok { + return fmt.Errorf("app user cannot see bucket") + } + return nil +} + +func policyJSON(bucket string) []byte { + return []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::` + bucket + `"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::` + bucket + `/*"] + } + ] +}`) +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..344d6847 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,169 @@ +package config + +import ( + "fmt" + "net" + "os" + "strconv" + "time" +) + +// Config — параметры процесса. Секреты приходят из окружения, не из образа. +type Config struct { + Host string + Port string + Env string + ShutdownTimeout time.Duration + DatabaseURL string + RedisURL string + SessionTTL time.Duration + DBMaxConns int32 + Argon2Time uint32 + Argon2Memory uint32 + Argon2Threads uint8 + InstanceID string + IdempotencyTTL time.Duration + S3Endpoint string + S3Region string + S3Bucket string + S3AccessKey string + S3SecretKey string + MaxUploadBytes int64 + MaxImagePixels int + ReconcileGrace time.Duration +} + +// Addr — адрес прослушивания. В контейнере это 0.0.0.0:8080, +// иначе published port Docker не попадёт в процесс. +func (c Config) Addr() string { + return net.JoinHostPort(c.Host, c.Port) +} + +// Load завершает запуск ошибкой, если значение вне допустимого диапазона. +func Load() (Config, error) { + var c Config + c.Host = getenv("APP_HOST", "0.0.0.0") + c.Port = getenv("APP_PORT", "8080") + c.Env = getenv("APP_ENV", "lab") + if c.Host == "" { + return Config{}, fmt.Errorf("APP_HOST is empty") + } + port, err := strconv.Atoi(c.Port) + if err != nil || port < 1 || port > 65535 { + return Config{}, fmt.Errorf("APP_PORT must be 1..65535") + } + + sec, err := nonNeg("SHUTDOWN_TIMEOUT_SECONDS", 8) + if err != nil { + return Config{}, err + } + if sec < 1 || sec > 10 { + return Config{}, fmt.Errorf("SHUTDOWN_TIMEOUT_SECONDS must be 1..10") + } + c.ShutdownTimeout = time.Duration(sec) * time.Second + + c.DatabaseURL = getenv("DATABASE_URL", "") + c.RedisURL = getenv("REDIS_URL", "") + if c.DatabaseURL == "" || c.RedisURL == "" { + return Config{}, fmt.Errorf("DATABASE_URL and REDIS_URL are required") + } + ttl, err := nonNeg("SESSION_TTL_SECONDS", 3600) + if err != nil { + return Config{}, err + } + if ttl < 1 || ttl > 7*24*3600 { + return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be 1..604800") + } + c.SessionTTL = time.Duration(ttl) * time.Second + + maxConns, err := nonNeg("DB_MAX_CONNS", 10) + if err != nil { + return Config{}, err + } + if maxConns < 1 || maxConns > 50 { + return Config{}, fmt.Errorf("DB_MAX_CONNS must be 1..50") + } + c.DBMaxConns = int32(maxConns) + + argonTime, err := nonNeg("ARGON2_TIME", 2) + if err != nil { + return Config{}, err + } + argonMemory, err := nonNeg("ARGON2_MEMORY_KIB", 19456) + if err != nil { + return Config{}, err + } + argonThreads, err := nonNeg("ARGON2_THREADS", 1) + if err != nil { + return Config{}, err + } + if argonTime < 1 || argonTime > 10 || argonMemory < 8 || argonMemory > 1<<20 || argonThreads < 1 || argonThreads > 8 { + return Config{}, fmt.Errorf("argon2 params are out of range") + } + c.Argon2Time = uint32(argonTime) + c.Argon2Memory = uint32(argonMemory) + c.Argon2Threads = uint8(argonThreads) + + c.InstanceID = getenv("INSTANCE_ID", "") + idemTTL, err := nonNeg("IDEMPOTENCY_TTL_SECONDS", 86400) + if err != nil { + return Config{}, err + } + if idemTTL < 86400 || idemTTL > 7*24*3600 { + return Config{}, fmt.Errorf("IDEMPOTENCY_TTL_SECONDS must be 86400..604800") + } + c.IdempotencyTTL = time.Duration(idemTTL) * time.Second + + c.S3Endpoint = getenv("S3_ENDPOINT", "") + c.S3Region = getenv("S3_REGION", "us-east-1") + c.S3Bucket = getenv("S3_BUCKET", "") + c.S3AccessKey = getenv("S3_ACCESS_KEY", "") + c.S3SecretKey = getenv("S3_SECRET_KEY", "") + if c.S3Endpoint == "" || c.S3Bucket == "" || c.S3AccessKey == "" || c.S3SecretKey == "" { + return Config{}, fmt.Errorf("S3_ENDPOINT, S3_BUCKET, S3_ACCESS_KEY and S3_SECRET_KEY are required") + } + upload, err := nonNeg("MAX_UPLOAD_BYTES", 10*1024*1024) + if err != nil { + return Config{}, err + } + if upload < 1 || upload > 10*1024*1024 { + return Config{}, fmt.Errorf("MAX_UPLOAD_BYTES must be 1..10485760") + } + c.MaxUploadBytes = int64(upload) + pixels, err := nonNeg("MAX_IMAGE_PIXELS", 40_000_000) + if err != nil { + return Config{}, err + } + if pixels < 1 || pixels > 40_000_000 { + return Config{}, fmt.Errorf("MAX_IMAGE_PIXELS must be 1..40000000") + } + c.MaxImagePixels = pixels + grace, err := nonNeg("RECONCILE_GRACE_SECONDS", 900) + if err != nil { + return Config{}, err + } + if grace > 24*3600 { + return Config{}, fmt.Errorf("RECONCILE_GRACE_SECONDS must be 0..86400") + } + c.ReconcileGrace = time.Duration(grace) * time.Second + return c, nil +} + +func getenv(key, def string) string { + if v, ok := os.LookupEnv(key); ok && v != "" { + return v + } + return def +} + +func nonNeg(key string, def int) (int, error) { + v, ok := os.LookupEnv(key) + if !ok || v == "" { + return def, nil + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return 0, fmt.Errorf("%s must be a non-negative integer", key) + } + return n, nil +} diff --git a/internal/imagecheck/image.go b/internal/imagecheck/image.go new file mode 100644 index 00000000..a5959b54 --- /dev/null +++ b/internal/imagecheck/image.go @@ -0,0 +1,190 @@ +// Проверка PNG/JPEG без распаковки всего растра в память. +// Размеры читаются из заголовка. Полный Decode выполняется только для небольших +// картинок, чтобы поймать битый поток. +package imagecheck + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "image" + _ "image/jpeg" + _ "image/png" +) + +const decodePixelLimit = 1_000_000 + +var ( + ErrFormat = errors.New("image format") + ErrPixels = errors.New("image pixels") +) + +var pngSig = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + +// Inspect сверяет заявленный MIME с байтами, проверяет структуру и лимит пикселей. +// declared - ровно image/png или image/jpeg, без параметров. +func Inspect(declared string, data []byte, maxPixels int) (width, height int, err error) { + if maxPixels < 1 { + return 0, 0, ErrFormat + } + detected := Detect(data) + if detected == "" || detected != declared { + return 0, 0, ErrFormat + } + switch detected { + case "image/png": + width, height, err = parsePNG(data) + case "image/jpeg": + width, height, err = parseJPEG(data) + default: + return 0, 0, ErrFormat + } + if err != nil { + return 0, 0, ErrFormat + } + if uint64(width)*uint64(height) > uint64(maxPixels) { + return 0, 0, ErrPixels + } + // Маленький файл декодируем целиком: битый IDAT/скан не должен стать ready. + // Выше порога остаётся проверка заголовка и CRC/маркеров, без буфера width*height*4. + if uint64(width)*uint64(height) <= decodePixelLimit { + if _, _, decErr := image.Decode(bytes.NewReader(data)); decErr != nil { + return 0, 0, ErrFormat + } + } + return width, height, nil +} + +// Detect смотрит только на сигнатуру, не на заголовок Content-Type. +func Detect(data []byte) string { + if len(data) >= len(pngSig) && bytes.Equal(data[:len(pngSig)], pngSig) { + return "image/png" + } + if len(data) >= 3 && data[0] == 0xff && data[1] == 0xd8 && data[2] == 0xff { + return "image/jpeg" + } + return "" +} + +func parsePNG(data []byte) (int, int, error) { + if len(data) < len(pngSig) || !bytes.Equal(data[:len(pngSig)], pngSig) { + return 0, 0, ErrFormat + } + i := len(pngSig) + first := true + var width, height int + for i < len(data) { + if i+12 > len(data) { + return 0, 0, ErrFormat + } + n64 := uint64(binary.BigEndian.Uint32(data[i : i+4])) + if n64 > uint64(len(data)-i-12) { + return 0, 0, ErrFormat + } + n := int(n64) + chunk := data[i+4 : i+8+n] + got := binary.BigEndian.Uint32(data[i+8+n : i+12+n]) + if crc32.ChecksumIEEE(chunk) != got { + return 0, 0, ErrFormat + } + kind := string(data[i+4 : i+8]) + if first { + if kind != "IHDR" || n != 13 { + return 0, 0, ErrFormat + } + width = int(binary.BigEndian.Uint32(data[i+8 : i+12])) + height = int(binary.BigEndian.Uint32(data[i+12 : i+16])) + if width < 1 || height < 1 { + return 0, 0, ErrFormat + } + first = false + } + i += 12 + n + if kind == "IEND" { + if n != 0 || i != len(data) { + return 0, 0, ErrFormat + } + return width, height, nil + } + } + return 0, 0, ErrFormat +} + +func parseJPEG(data []byte) (int, int, error) { + if len(data) < 4 || data[0] != 0xff || data[1] != 0xd8 { + return 0, 0, ErrFormat + } + i := 2 + var width, height int + sawSOF := false + sawEOI := false + for i < len(data) { + if data[i] != 0xff { + return 0, 0, ErrFormat + } + for i < len(data) && data[i] == 0xff { + i++ + } + if i >= len(data) { + return 0, 0, ErrFormat + } + marker := data[i] + i++ + switch { + case marker == 0xd9: + sawEOI = true + case marker == 0x01 || (marker >= 0xd0 && marker <= 0xd7): + continue + case marker == 0xda: + // Энтропийные данные до следующего маркера. + for i < len(data) { + if data[i] != 0xff { + i++ + continue + } + if i+1 >= len(data) { + return 0, 0, ErrFormat + } + if data[i+1] == 0x00 { + i += 2 + continue + } + break + } + continue + default: + if i+2 > len(data) { + return 0, 0, ErrFormat + } + seglen := int(data[i])<<8 | int(data[i+1]) + if seglen < 2 || i+seglen > len(data) { + return 0, 0, ErrFormat + } + if isSOF(marker) { + if seglen < 7 { + return 0, 0, ErrFormat + } + height = int(data[i+3])<<8 | int(data[i+4]) + width = int(data[i+5])<<8 | int(data[i+6]) + sawSOF = true + } + i += seglen + continue + } + break + } + if !sawSOF || !sawEOI || width < 1 || height < 1 { + return 0, 0, ErrFormat + } + return width, height, nil +} + +func isSOF(marker byte) bool { + switch marker { + case 0xc0, 0xc1, 0xc2: + return true + default: + return false + } +} diff --git a/internal/imagecheck/image_test.go b/internal/imagecheck/image_test.go new file mode 100644 index 00000000..b61b8fb6 --- /dev/null +++ b/internal/imagecheck/image_test.go @@ -0,0 +1,114 @@ +package imagecheck + +import ( + "bytes" + "hash/crc32" + "image" + "image/color" + "image/jpeg" + "image/png" + "os" + "testing" +) + +func tinyPNG(t *testing.T, w, h int) []byte { + t.Helper() + img := image.NewNRGBA(image.Rect(0, 0, w, h)) + img.Set(0, 0, color.NRGBA{R: 10, G: 20, B: 30, A: 255}) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func tinyJPEG(t *testing.T) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, 8, 4)) + var buf bytes.Buffer + if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 80}); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestInspectAcceptsEncodedImages(t *testing.T) { + pngBytes := tinyPNG(t, 3, 2) + w, h, err := Inspect("image/png", pngBytes, 40_000_000) + if err != nil || w != 3 || h != 2 { + t.Fatalf("png: %d %d %v", w, h, err) + } + jpg := tinyJPEG(t) + w, h, err = Inspect("image/jpeg", jpg, 40_000_000) + if err != nil || w != 8 || h != 4 { + t.Fatalf("jpeg: %d %d %v", w, h, err) + } +} + +func TestInspectRejectsMIMESwapAndDamage(t *testing.T) { + pngBytes := tinyPNG(t, 2, 2) + if _, _, err := Inspect("image/jpeg", pngBytes, 40_000_000); err != ErrFormat { + t.Fatalf("png declared as jpeg: %v", err) + } + if _, _, err := Inspect("image/png", tinyJPEG(t), 40_000_000); err != ErrFormat { + t.Fatalf("jpeg declared as png: %v", err) + } + if _, _, err := Inspect("image/png", []byte("not an image"), 40_000_000); err != ErrFormat { + t.Fatalf("text: %v", err) + } + cut := pngBytes[:len(pngBytes)/2] + if _, _, err := Inspect("image/png", cut, 40_000_000); err != ErrFormat { + t.Fatalf("truncated: %v", err) + } + flipped := append([]byte(nil), pngBytes...) + flipped[len(flipped)-20] ^= 0xff + if _, _, err := Inspect("image/png", flipped, 40_000_000); err != ErrFormat { + t.Fatalf("flipped byte: %v", err) + } +} + +func TestInspectPixelLimitBeforeDecode(t *testing.T) { + // 20000×2001 = 40_020_000 > 40_000_000. Файл маленький: лимит смотрит IHDR, не растр. + huge := pngWithSize(t, 20000, 2001) + if _, _, err := Inspect("image/png", huge, 40_000_000); err != ErrPixels { + t.Fatalf("pixels: %v", err) + } +} + +func pngWithSize(t *testing.T, w, h int) []byte { + t.Helper() + // Валидный минимальный PNG с подменёнными размерами и пересчитанным CRC IHDR + // не обязателен: достаточно корректного CRC, Decode для огромного размера не вызывается. + base := tinyPNG(t, 1, 1) + // IHDR width/height лежат сразу после сигнатуры и длины/типа: смещение 16 и 20. + out := append([]byte(nil), base...) + out[16] = byte(w >> 24) + out[17] = byte(w >> 16) + out[18] = byte(w >> 8) + out[19] = byte(w) + out[20] = byte(h >> 24) + out[21] = byte(h >> 16) + out[22] = byte(h >> 8) + out[23] = byte(h) + // CRC IHDR: байты 12..28 type+data, CRC по смещению 29. + sum := crc32.ChecksumIEEE(out[12:29]) + out[29] = byte(sum >> 24) + out[30] = byte(sum >> 16) + out[31] = byte(sum >> 8) + out[32] = byte(sum) + return out +} + +func TestFixtureSample(t *testing.T) { + b, err := os.ReadFile("../../tests/fixtures/sample.png") + if err != nil { + t.Fatal(err) + } + w, h, err := Inspect("image/png", b, 40_000_000) + if err != nil || w != 320 || h != 200 { + t.Fatalf("fixture: %d %d %v", w, h, err) + } + if _, _, err := Inspect("image/jpeg", b, 40_000_000); err != ErrFormat { + t.Fatalf("fixture mime: %v", err) + } +} diff --git a/internal/migrate/migrate.go b/internal/migrate/migrate.go new file mode 100644 index 00000000..0e99ebcc --- /dev/null +++ b/internal/migrate/migrate.go @@ -0,0 +1,82 @@ +// применяет SQL из бинарника ровно один раз на файл. +package migrate + +import ( + "context" + "fmt" + "io/fs" + "sort" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/migrations" +) + +// advisoryLock сериализует миграции между процессами. Номер постоянный для этого приложения. +const advisoryLock int64 = 748392016 + +// Apply создаёт schema_migrations и выполняет ещё не записанные файлы. +// Замок держится на одном соединении: пул сам по себе не привязывает блокировку к сессии. +func Apply(ctx context.Context, pool *pgxpool.Pool) error { + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, `SELECT pg_advisory_lock($1)`, advisoryLock); err != nil { + return fmt.Errorf("advisory lock: %w", err) + } + defer func() { + _, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, advisoryLock) + }() + + if _, err := conn.Exec(ctx, ` + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + )`); err != nil { + return fmt.Errorf("schema_migrations: %w", err) + } + entries, err := fs.ReadDir(migrations.FS, ".") + if err != nil { + return err + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".sql") { + continue + } + names = append(names, entry.Name()) + } + sort.Strings(names) + for _, name := range names { + var applied bool + if err := conn.QueryRow(ctx, `SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1)`, name).Scan(&applied); err != nil { + return err + } + if applied { + continue + } + body, err := fs.ReadFile(migrations.FS, name) + if err != nil { + return err + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("%s: %w", name, err) + } + if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version) VALUES ($1)`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} diff --git a/internal/password/password.go b/internal/password/password.go new file mode 100644 index 00000000..768dfa0f --- /dev/null +++ b/internal/password/password.go @@ -0,0 +1,63 @@ +// Package password хеширует пароли Argon2id. +package password + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +// Params — стоимость KDF. Значения пишутся в саму строку хеша, +// поэтому старые пароли проверяются даже после смены настроек. +type Params struct { + Time uint32 + Memory uint32 + Threads uint8 +} + +// Hash возвращает PHC-строку. +func (p Params) Hash(plain string) (string, error) { + if p.Time < 1 || p.Threads < 1 || p.Memory < 8*uint32(p.Threads) { + return "", fmt.Errorf("argon2 params are too small") + } + salt := make([]byte, 16) + if _, err := rand.Read(salt); err != nil { + return "", err + } + sum := argon2.IDKey([]byte(plain), salt, p.Time, p.Memory, p.Threads, 32) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, + p.Memory, p.Time, p.Threads, + base64.RawStdEncoding.EncodeToString(salt), + base64.RawStdEncoding.EncodeToString(sum), + ), nil +} + +// Verify сравнивает пароль с PHC-строкой. Параметры берутся из строки, не из текущего конфига. +func Verify(plain, encoded string) bool { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" || !strings.HasPrefix(parts[2], "v=") { + return false + } + var memory, timeCost, threads int + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &memory, &timeCost, &threads); err != nil { + return false + } + if memory < 8 || timeCost < 1 || threads < 1 || threads > 255 { + return false + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil || len(salt) == 0 { + return false + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil || len(want) == 0 { + return false + } + got := argon2.IDKey([]byte(plain), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want))) + return subtle.ConstantTimeCompare(got, want) == 1 +} diff --git a/internal/password/password_test.go b/internal/password/password_test.go new file mode 100644 index 00000000..51b654fd --- /dev/null +++ b/internal/password/password_test.go @@ -0,0 +1,23 @@ +package password + +import "testing" + +func TestArgon2idRoundTrip(t *testing.T) { + p := Params{Time: 1, Memory: 8, Threads: 1} + encoded, err := p.Hash("correct-horse-1") + if err != nil { + t.Fatal(err) + } + if encoded == "correct-horse-1" || len(encoded) < 20 { + t.Fatalf("hash is not a PHC string: %s", encoded) + } + if !Verify("correct-horse-1", encoded) { + t.Fatal("matching password rejected") + } + if Verify("other-password-1", encoded) { + t.Fatal("wrong password accepted") + } + if Verify("correct-horse-1", "sha256:deadbeef") { + t.Fatal("non-argon string accepted") + } +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 00000000..e7485924 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,115 @@ +// Сверка PostgreSQL и bucket. +// Удаляются только старые расхождения: объект без строки и строка без объекта, +package reconcile + +import ( + "context" + "time" + + "messenger/internal/blob" + "messenger/internal/store" +) + +// Meta - чтение и безопасное удаление метаданных. Удаление само проверяет ссылки и возраст. +type Meta interface { + AttachmentRefs(ctx context.Context) ([]store.AttachmentRef, error) + DeleteUnreferencedAttachment(ctx context.Context, id string, olderThan time.Time) (bool, error) +} + +// Summary - наблюдаемый результат одного прохода. +type Summary struct { + Mode string `json:"mode"` + GraceSeconds int `json:"grace_seconds"` + Cutoff string `json:"cutoff"` + DBRows int `json:"db_rows"` + S3Objects int `json:"s3_objects"` + ListPages int `json:"list_pages"` + Pairs int `json:"pairs"` + OrphanFresh int `json:"orphan_fresh"` + OrphanStale int `json:"orphan_stale"` + MissingFresh int `json:"missing_fresh"` + MissingReferenced int `json:"missing_referenced"` + MissingStaleUnreferenced int `json:"missing_stale_unreferenced"` + DeletedObjects int `json:"deleted_objects"` + DeletedRows int `json:"deleted_rows"` + DurationMs int64 `json:"duration_ms"` +} + +// Run сравнивает множество ключей. Стоимость - один листинг bucket и один запрос метаданных, +// без HEAD на каждый объект. +func Run(ctx context.Context, meta Meta, objects blob.Store, grace time.Duration, apply bool) (Summary, error) { + start := time.Now() + if grace < 0 { + grace = 0 + } + cutoff := time.Now().UTC().Add(-grace) + rows, err := meta.AttachmentRefs(ctx) + if err != nil { + return Summary{}, err + } + listed, pages, err := objects.List(ctx) + if err != nil { + return Summary{}, err + } + byKey := make(map[string]store.AttachmentRef, len(rows)) + for _, row := range rows { + byKey[row.ObjectKey] = row + } + seen := make(map[string]struct{}, len(listed)) + sum := Summary{ + Mode: "dry-run", + GraceSeconds: int(grace / time.Second), + Cutoff: cutoff.Format(time.RFC3339), + DBRows: len(rows), + S3Objects: len(listed), + ListPages: pages, + } + if apply { + sum.Mode = "apply" + } + for _, obj := range listed { + seen[obj.Key] = struct{}{} + if _, ok := byKey[obj.Key]; ok { + sum.Pairs++ + continue + } + if !obj.Modified.UTC().Before(cutoff) { + sum.OrphanFresh++ + continue + } + sum.OrphanStale++ + if !apply { + continue + } + if err := objects.Delete(ctx, obj.Key); err != nil { + return Summary{}, err + } + sum.DeletedObjects++ + } + for _, row := range rows { + if _, ok := seen[row.ObjectKey]; ok { + continue + } + if row.Referenced { + sum.MissingReferenced++ + continue + } + if !row.CreatedAt.UTC().Before(cutoff) { + sum.MissingFresh++ + continue + } + sum.MissingStaleUnreferenced++ + if !apply { + continue + } + deleted, err := meta.DeleteUnreferencedAttachment(ctx, row.ID, cutoff) + if err != nil { + return Summary{}, err + } + if deleted { + sum.DeletedRows++ + } + } + sum.DurationMs = time.Since(start).Milliseconds() + return sum, nil +} diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go new file mode 100644 index 00000000..3b85fd78 --- /dev/null +++ b/internal/reconcile/reconcile_test.go @@ -0,0 +1,98 @@ +package reconcile + +import ( + "context" + "testing" + "time" + + "messenger/internal/blob" + "messenger/internal/store" +) + +func TestReconcileKeepsFreshAndReferenced(t *testing.T) { + ctx := context.Background() + meta := store.New() + objects := blob.NewMemory() + old := time.Now().UTC().Add(-time.Hour) + fresh := time.Now().UTC() + + mustPut := func(key string, at time.Time) { + t.Helper() + if err := objects.PutAt(ctx, key, []byte("x"), at); err != nil { + t.Fatal(err) + } + } + mustRow := func(key string, at time.Time, referenced bool) string { + t.Helper() + row, err := meta.InsertAttachment(ctx, store.Attachment{ + ChatID: "chat", OwnerID: "user", Filename: "a.png", ContentType: "image/png", + SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ObjectKey: key, Width: 1, Height: 1, CreatedAt: at, + }) + if err != nil { + t.Fatal(err) + } + if referenced { + meta.MarkReferenced(row.ID) + } + return row.ID + } + + mustPut("orphan-old", old) + mustPut("orphan-fresh", fresh) + mustPut("pair-fresh", fresh) + refID := mustRow("missing-ref", old, true) + staleID := mustRow("missing-stale", old, false) + freshID := mustRow("missing-fresh", fresh, false) + pairID := mustRow("pair-fresh", fresh, false) + + grace := 15 * time.Minute + dry, err := Run(ctx, meta, objects, grace, false) + if err != nil { + t.Fatal(err) + } + if dry.Mode != "dry-run" || dry.DeletedObjects != 0 || dry.DeletedRows != 0 { + t.Fatalf("dry-run mutated: %+v", dry) + } + if dry.OrphanStale != 1 || dry.OrphanFresh != 1 || dry.MissingReferenced != 1 || dry.MissingStaleUnreferenced != 1 || dry.MissingFresh != 1 || dry.Pairs != 1 { + t.Fatalf("classification: %+v", dry) + } + again, err := Run(ctx, meta, objects, grace, false) + if err != nil || again.OrphanStale != dry.OrphanStale || again.MissingStaleUnreferenced != dry.MissingStaleUnreferenced { + t.Fatalf("second dry-run: %+v %v", again, err) + } + + applied, err := Run(ctx, meta, objects, grace, true) + if err != nil { + t.Fatal(err) + } + if applied.DeletedObjects != 1 || applied.DeletedRows != 1 { + t.Fatalf("apply: %+v", applied) + } + if _, err := objects.Get(ctx, "orphan-old"); err != blob.ErrNotFound { + t.Fatalf("old orphan still present: %v", err) + } + if _, err := objects.Get(ctx, "orphan-fresh"); err != nil { + t.Fatalf("fresh orphan deleted: %v", err) + } + if _, err := meta.AttachmentByID(ctx, refID); err != nil { + t.Fatalf("referenced row deleted: %v", err) + } + if _, err := meta.AttachmentByID(ctx, freshID); err != nil { + t.Fatalf("fresh row deleted: %v", err) + } + if _, err := meta.AttachmentByID(ctx, pairID); err != nil { + t.Fatalf("pair deleted: %v", err) + } + if _, err := meta.AttachmentByID(ctx, staleID); err != store.ErrNotFound { + t.Fatalf("stale row remains: %v", err) + } + + clean, err := Run(ctx, meta, objects, grace, true) + if err != nil { + t.Fatal(err) + } + if clean.DeletedObjects != 0 || clean.DeletedRows != 0 || clean.OrphanStale != 0 || clean.MissingStaleUnreferenced != 0 { + t.Fatalf("rerun not idempotent: %+v", clean) + } +} diff --git a/internal/session/redis_test.go b/internal/session/redis_test.go new file mode 100644 index 00000000..a7108f9b --- /dev/null +++ b/internal/session/redis_test.go @@ -0,0 +1,77 @@ +package session + +import ( + "context" + "errors" + "os" + "testing" + "time" +) + +func TestMemoryExpiryAndRevoke(t *testing.T) { + s := NewMemory() + ctx := context.Background() + exp := time.Now().Add(30 * time.Millisecond) + if err := s.Create(ctx, "token-value-with-enough-entropy-xx", "user-1", exp); err != nil { + t.Fatal(err) + } + if _, _, err := s.Lookup(ctx, "token-value-with-enough-entropy-xx"); err != nil { + t.Fatal(err) + } + time.Sleep(40 * time.Millisecond) + if _, _, err := s.Lookup(ctx, "token-value-with-enough-entropy-xx"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expired: %v", err) + } + exp = time.Now().Add(time.Minute) + if err := s.Create(ctx, "token-value-with-enough-entropy-yy", "user-1", exp); err != nil { + t.Fatal(err) + } + if err := s.Revoke(ctx, "token-value-with-enough-entropy-yy"); err != nil { + t.Fatal(err) + } + if _, _, err := s.Lookup(ctx, "token-value-with-enough-entropy-yy"); !errors.Is(err, ErrNotFound) { + t.Fatalf("revoked: %v", err) + } +} + +func TestRedisDown(t *testing.T) { + s, err := OpenRedis("redis://127.0.0.1:1/0") + if err != nil { + t.Fatal(err) + } + defer s.Close() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := s.Ping(ctx); !errors.Is(err, ErrUnavailable) { + t.Fatalf("ping: %v", err) + } + if err := s.Create(ctx, "token", "user", time.Now().Add(time.Minute)); !errors.Is(err, ErrUnavailable) { + t.Fatalf("create: %v", err) + } +} + +func TestIntegrationRedisRoundTrip(t *testing.T) { + raw := os.Getenv("REDIS_URL") + if raw == "" { + t.Skip("REDIS_URL is not set") + } + s, err := OpenRedis(raw) + if err != nil { + t.Fatal(err) + } + defer s.Close() + ctx := context.Background() + token := "integration-token-" + time.Now().UTC().Format("150405.000000") + exp := time.Now().UTC().Add(2 * time.Second).Truncate(time.Microsecond) + if err := s.Create(ctx, token, "11111111-1111-4111-8111-111111111111", exp); err != nil { + t.Fatal(err) + } + id, got, err := s.Lookup(ctx, token) + if err != nil || id != "11111111-1111-4111-8111-111111111111" || !got.Equal(exp) { + t.Fatalf("lookup id=%s exp=%s err=%v", id, got, err) + } + time.Sleep(2200 * time.Millisecond) + if _, _, err := s.Lookup(ctx, token); !errors.Is(err, ErrNotFound) { + t.Fatalf("ttl did not expire: %v", err) + } +} diff --git a/internal/session/session.go b/internal/session/session.go new file mode 100644 index 00000000..367ccb29 --- /dev/null +++ b/internal/session/session.go @@ -0,0 +1,173 @@ +// Package session хранит непрозрачные Bearer-сессии. +// В стенде это Redis: ключ живёт ровно до expires_at, рестарт TTL не сдвигает. +package session + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "sync" + "time" + + "github.com/redis/go-redis/v9" +) + +var ( + ErrNotFound = errors.New("session not found") + ErrUnavailable = errors.New("session store unavailable") +) + +// Store — срок жизни сессии задаёт вызывающий, хранилище его не продлевает. +type Store interface { + Create(ctx context.Context, token, userID string, expiresAt time.Time) error + Lookup(ctx context.Context, token string) (userID string, expiresAt time.Time, err error) + Revoke(ctx context.Context, token string) error + Ping(ctx context.Context) error + Close() error +} + +type record struct { + UserID string `json:"user_id"` + ExpUnix int64 `json:"exp_unix"` +} + +// key — хеш токена. Сам токен в Redis не пишется: утечка дампа не равна утечке Bearer. +func key(token string) string { + sum := sha256.Sum256([]byte(token)) + return "sess:" + hex.EncodeToString(sum[:]) +} + +// Memory — сессии процесса. Нужна юнит-тестам без Docker. +type Memory struct { + mu sync.Mutex + items map[string]record +} + +func NewMemory() *Memory { + return &Memory{items: make(map[string]record)} +} + +func (m *Memory) Create(_ context.Context, token, userID string, expiresAt time.Time) error { + if token == "" || userID == "" { + return ErrNotFound + } + m.mu.Lock() + defer m.mu.Unlock() + m.items[key(token)] = record{UserID: userID, ExpUnix: expiresAt.UTC().UnixMicro()} + return nil +} + +func (m *Memory) Lookup(_ context.Context, token string) (string, time.Time, error) { + m.mu.Lock() + defer m.mu.Unlock() + rec, ok := m.items[key(token)] + if !ok { + return "", time.Time{}, ErrNotFound + } + exp := time.UnixMicro(rec.ExpUnix).UTC() + if !time.Now().Before(exp) { + delete(m.items, key(token)) + return "", time.Time{}, ErrNotFound + } + return rec.UserID, exp, nil +} + +func (m *Memory) Revoke(_ context.Context, token string) error { + m.mu.Lock() + defer m.mu.Unlock() + k := key(token) + if _, ok := m.items[k]; !ok { + return ErrNotFound + } + delete(m.items, k) + return nil +} + +func (m *Memory) Ping(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return ErrUnavailable + } + return nil +} + +func (m *Memory) Close() error { return nil } + +// Redis — единственное хранилище сессий стенда. +type Redis struct { + client *redis.Client +} + +func OpenRedis(rawURL string) (*Redis, error) { + opt, err := redis.ParseURL(rawURL) + if err != nil { + return nil, err + } + opt.DialTimeout = 500 * time.Millisecond + opt.ReadTimeout = 500 * time.Millisecond + opt.WriteTimeout = 500 * time.Millisecond + opt.PoolSize = 10 + opt.MinIdleConns = 1 + return &Redis{client: redis.NewClient(opt)}, nil +} + +func (r *Redis) Create(ctx context.Context, token, userID string, expiresAt time.Time) error { + if token == "" || userID == "" { + return ErrNotFound + } + ttl := time.Until(expiresAt) + if ttl < time.Millisecond { + return ErrNotFound + } + body, err := json.Marshal(record{UserID: userID, ExpUnix: expiresAt.UTC().UnixMicro()}) + if err != nil { + return err + } + if err := r.client.Set(ctx, key(token), body, ttl).Err(); err != nil { + return ErrUnavailable + } + return nil +} + +func (r *Redis) Lookup(ctx context.Context, token string) (string, time.Time, error) { + raw, err := r.client.Get(ctx, key(token)).Bytes() + if errors.Is(err, redis.Nil) { + return "", time.Time{}, ErrNotFound + } + if err != nil { + return "", time.Time{}, ErrUnavailable + } + var rec record + if err := json.Unmarshal(raw, &rec); err != nil || rec.UserID == "" { + return "", time.Time{}, ErrNotFound + } + exp := time.UnixMicro(rec.ExpUnix).UTC() + if !time.Now().Before(exp) { + _ = r.client.Del(ctx, key(token)).Err() + return "", time.Time{}, ErrNotFound + } + return rec.UserID, exp, nil +} + +func (r *Redis) Revoke(ctx context.Context, token string) error { + n, err := r.client.Del(ctx, key(token)).Result() + if err != nil { + return ErrUnavailable + } + if n == 0 { + return ErrNotFound + } + return nil +} + +func (r *Redis) Ping(ctx context.Context) error { + if err := r.client.Ping(ctx).Err(); err != nil { + return ErrUnavailable + } + return nil +} + +func (r *Redis) Close() error { + return r.client.Close() +} diff --git a/internal/store/attachments.go b/internal/store/attachments.go new file mode 100644 index 00000000..c3d92efe --- /dev/null +++ b/internal/store/attachments.go @@ -0,0 +1,286 @@ +package store + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5" +) + +func (p *PG) InsertAttachment(ctx context.Context, row Attachment) (Attachment, error) { + if row.ID == "" { + row.ID = newID() + } + now := time.Now().UTC().Truncate(time.Microsecond) + if row.CreatedAt.IsZero() { + row.CreatedAt = now + } else { + row.CreatedAt = row.CreatedAt.UTC().Truncate(time.Microsecond) + } + row.UpdatedAt = row.CreatedAt + row.Status = "ready" + _, err := p.pool.Exec(ctx, ` + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, width, height, status, created_at, updated_at + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 + )`, + row.ID, row.ChatID, row.OwnerID, row.Filename, row.ContentType, row.SizeBytes, row.SHA256, + row.ObjectKey, row.Width, row.Height, row.Status, row.CreatedAt, row.UpdatedAt) + if err != nil { + return Attachment{}, mapDB(err) + } + return row, nil +} + +func (p *PG) AttachmentVisible(ctx context.Context, id, userID string) (Attachment, error) { + var row Attachment + err := p.pool.QueryRow(ctx, ` + SELECT a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, + a.size_bytes, a.sha256, a.object_key, a.width, a.height, a.status, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1`, id, userID).Scan( + &row.ID, &row.ChatID, &row.OwnerID, &row.Filename, &row.ContentType, + &row.SizeBytes, &row.SHA256, &row.ObjectKey, &row.Width, &row.Height, &row.Status, + &row.CreatedAt, &row.UpdatedAt) + if err != nil { + return Attachment{}, mapDB(err) + } + row.CreatedAt = row.CreatedAt.UTC() + row.UpdatedAt = row.UpdatedAt.UTC() + return row, nil +} + +func (p *PG) AttachmentRefs(ctx context.Context) ([]AttachmentRef, error) { + rows, err := p.pool.Query(ctx, ` + SELECT a.id::text, a.object_key, a.created_at, + EXISTS(SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id) + FROM attachments a`) + if err != nil { + return nil, mapDB(err) + } + defer rows.Close() + out := make([]AttachmentRef, 0) + for rows.Next() { + var row AttachmentRef + if err := rows.Scan(&row.ID, &row.ObjectKey, &row.CreatedAt, &row.Referenced); err != nil { + return nil, mapDB(err) + } + row.CreatedAt = row.CreatedAt.UTC() + out = append(out, row) + } + if err := rows.Err(); err != nil { + return nil, mapDB(err) + } + return out, nil +} + +func (p *PG) DeleteUnreferencedAttachment(ctx context.Context, id string, olderThan time.Time) (bool, error) { + tag, err := p.pool.Exec(ctx, ` + DELETE FROM attachments a + WHERE a.id = $1 + AND a.created_at < $2 + AND NOT EXISTS (SELECT 1 FROM message_attachments m WHERE m.attachment_id = a.id)`, id, olderThan) + if err != nil { + return false, mapDB(err) + } + return tag.RowsAffected() == 1, nil +} + +func bindAttachments(ctx context.Context, tx pgx.Tx, messageID, chatID string, ids []string) error { + for i, id := range ids { + var gotChat, status string + err := tx.QueryRow(ctx, ` + SELECT chat_id::text, status FROM attachments WHERE id = $1`, id).Scan(&gotChat, &status) + if err != nil { + if mapDB(err) == ErrNotFound { + return ErrNotFound + } + return err + } + if gotChat != chatID { + return ErrNotFound + } + if status != "ready" { + return ErrNotReady + } + if _, err := tx.Exec(ctx, ` + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, $3)`, messageID, id, i); err != nil { + return err + } + } + return nil +} + +func attachmentIDsOf(ctx context.Context, q queryer, messageID string) ([]string, error) { + rows, err := q.Query(ctx, ` + SELECT attachment_id::text + FROM message_attachments + WHERE message_id = $1 + ORDER BY position`, messageID) + if err != nil { + return nil, mapDB(err) + } + defer rows.Close() + ids := make([]string, 0) + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, mapDB(err) + } + ids = append(ids, id) + } + if err := rows.Err(); err != nil { + return nil, mapDB(err) + } + return ids, nil +} + +func fillAttachmentIDs(ctx context.Context, q queryer, items []Message) error { + if len(items) == 0 { + return nil + } + ids := make([]string, len(items)) + for i := range items { + ids[i] = items[i].ID + items[i].AttachmentIDs = []string{} + } + rows, err := q.Query(ctx, ` + SELECT message_id::text, attachment_id::text + FROM message_attachments + WHERE message_id::text = ANY($1::text[]) + ORDER BY message_id, position`, ids) + if err != nil { + return mapDB(err) + } + defer rows.Close() + grouped := map[string][]string{} + for rows.Next() { + var messageID, attachmentID string + if err := rows.Scan(&messageID, &attachmentID); err != nil { + return mapDB(err) + } + grouped[messageID] = append(grouped[messageID], attachmentID) + } + if err := rows.Err(); err != nil { + return mapDB(err) + } + for i := range items { + if got, ok := grouped[items[i].ID]; ok { + items[i].AttachmentIDs = got + } + } + return nil +} + +func (s *Memory) InsertAttachment(ctx context.Context, row Attachment) (Attachment, error) { + if err := ctx.Err(); err != nil { + return Attachment{}, ErrUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + if row.ID == "" { + row.ID = newID() + } + now := s.nextTimeLocked() + if row.CreatedAt.IsZero() { + row.CreatedAt = now + } else { + row.CreatedAt = row.CreatedAt.UTC() + } + row.UpdatedAt = row.CreatedAt + if row.Status == "" { + row.Status = "ready" + } + s.attachments[row.ID] = row + return row, nil +} + +func (s *Memory) AttachmentVisible(ctx context.Context, id, userID string) (Attachment, error) { + if err := ctx.Err(); err != nil { + return Attachment{}, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + row, ok := s.attachments[id] + if !ok { + return Attachment{}, ErrNotFound + } + rec, ok := s.chats[row.ChatID] + if !ok { + return Attachment{}, ErrNotFound + } + if _, member := rec.members[userID]; !member { + return Attachment{}, ErrNotFound + } + return row, nil +} + +func (s *Memory) AttachmentRefs(ctx context.Context) ([]AttachmentRef, error) { + if err := ctx.Err(); err != nil { + return nil, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + out := make([]AttachmentRef, 0, len(s.attachments)) + for _, row := range s.attachments { + out = append(out, AttachmentRef{ + ID: row.ID, ObjectKey: row.ObjectKey, CreatedAt: row.CreatedAt, Referenced: s.referencedLocked(row.ID), + }) + } + return out, nil +} + +func (s *Memory) DeleteUnreferencedAttachment(ctx context.Context, id string, olderThan time.Time) (bool, error) { + if err := ctx.Err(); err != nil { + return false, ErrUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + row, ok := s.attachments[id] + if !ok || s.referencedLocked(row.ID) || !row.CreatedAt.Before(olderThan) { + return false, nil + } + delete(s.attachments, id) + return true, nil +} + +// MarkReferenced нужен тесту сверки: строка считается используемой без сообщения. +func (s *Memory) MarkReferenced(id string) { + s.mu.Lock() + s.referenced[id] = struct{}{} + s.mu.Unlock() +} + +// AttachmentByID читает строку без проверки участника. Только для тестов сверки. +func (s *Memory) AttachmentByID(ctx context.Context, id string) (Attachment, error) { + if err := ctx.Err(); err != nil { + return Attachment{}, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + row, ok := s.attachments[id] + if !ok { + return Attachment{}, ErrNotFound + } + return row, nil +} + +func (s *Memory) referencedLocked(id string) bool { + if _, ok := s.referenced[id]; ok { + return true + } + for _, msgs := range s.messages { + for _, m := range msgs { + for _, aid := range m.AttachmentIDs { + if aid == id { + return true + } + } + } + } + return false +} diff --git a/internal/store/postgres.go b/internal/store/postgres.go new file mode 100644 index 00000000..658c0ed9 --- /dev/null +++ b/internal/store/postgres.go @@ -0,0 +1,473 @@ +package store + +import ( + "context" + "errors" + "net" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +// PG - источник истины для пользователей, чатов и сообщений. +// Запросы только с параметрами. Пул ограничен, у сессии соединения есть statement_timeout. +type PG struct { + pool *pgxpool.Pool + idemTTL time.Duration +} + +func Open(ctx context.Context, databaseURL string, maxConns int32) (*PG, error) { + cfg, err := pgxpool.ParseConfig(databaseURL) + if err != nil { + return nil, err + } + if maxConns < 1 { + maxConns = 10 + } + cfg.MaxConns = maxConns + cfg.MinConns = 1 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.HealthCheckPeriod = 30 * time.Second + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + cfg.ConnConfig.RuntimeParams["application_name"] = "messenger" + cfg.ConnConfig.RuntimeParams["statement_timeout"] = "5000" + cfg.ConnConfig.RuntimeParams["lock_timeout"] = "3000" + cfg.ConnConfig.RuntimeParams["idle_in_transaction_session_timeout"] = "10000" + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + pingCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, err + } + return &PG{pool: pool, idemTTL: 24 * time.Hour}, nil +} + +// SetIdempotencyTTL задаёт, сколько хранить результат POST сообщения. +func (p *PG) SetIdempotencyTTL(ttl time.Duration) { + if ttl > 0 { + p.idemTTL = ttl + } +} + +func (p *PG) Close() { p.pool.Close() } + +func (p *PG) Pool() *pgxpool.Pool { return p.pool } + +func (p *PG) Ping(ctx context.Context) error { + return mapDB(p.pool.Ping(ctx)) +} + +func (p *PG) CreateUser(ctx context.Context, username, displayName, passwordHash string) (User, error) { + u := User{ + ID: newID(), + Username: username, + DisplayName: displayName, + CreatedAt: time.Now().UTC().Truncate(time.Microsecond), + } + _, err := p.pool.Exec(ctx, ` + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5)`, + u.ID, u.Username, u.DisplayName, passwordHash, u.CreatedAt) + if err != nil { + return User{}, mapDB(err) + } + return u, nil +} + +func (p *PG) User(ctx context.Context, id string) (User, error) { + row := p.pool.QueryRow(ctx, ` + SELECT id::text, username, display_name, created_at + FROM users WHERE id = $1`, id) + return scanUser(row) +} + +func (p *PG) UserByUsername(ctx context.Context, username string) (User, string, error) { + var u User + var hash string + err := p.pool.QueryRow(ctx, ` + SELECT id::text, username, display_name, created_at, password_hash + FROM users WHERE username = $1`, username).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt, &hash) + if err != nil { + return User{}, "", mapDB(err) + } + u.CreatedAt = u.CreatedAt.UTC() + return u, hash, nil +} + +// CreateChat пишет чат и всех участников в одной транзакции. +// Если кого-то из списка нет, COMMIT не случается и строка чата не остаётся. +func (p *PG) CreateChat(ctx context.Context, title string, memberIDs []string, caller string) (Chat, error) { + seen := make(map[string]struct{}, len(memberIDs)) + for _, id := range memberIDs { + if _, dup := seen[id]; dup { + return Chat{}, ErrInvalid + } + seen[id] = struct{}{} + } + if _, ok := seen[caller]; !ok { + return Chat{}, ErrInvalid + } + tx, err := p.pool.Begin(ctx) + if err != nil { + return Chat{}, mapDB(err) + } + defer func() { _ = tx.Rollback(ctx) }() + + c := Chat{ + ID: newID(), + Title: title, + MemberIDs: append([]string(nil), memberIDs...), + CreatedAt: time.Now().UTC().Truncate(time.Microsecond), + } + if _, err := tx.Exec(ctx, `INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)`, c.ID, c.Title, c.CreatedAt); err != nil { + return Chat{}, mapDB(err) + } + var n int + if err := tx.QueryRow(ctx, `SELECT count(*) FROM users WHERE id::text = ANY($1::text[])`, memberIDs).Scan(&n); err != nil { + return Chat{}, mapDB(err) + } + if n != len(memberIDs) { + return Chat{}, ErrInvalid + } + for i, id := range memberIDs { + if _, err := tx.Exec(ctx, ` + INSERT INTO chat_members (chat_id, user_id, position) VALUES ($1, $2, $3)`, + c.ID, id, i); err != nil { + return Chat{}, mapDB(err) + } + } + if err := tx.Commit(ctx); err != nil { + return Chat{}, mapDB(err) + } + return c, nil +} + +func (p *PG) ChatFor(ctx context.Context, chatID, userID string) (Chat, error) { + var c Chat + err := p.pool.QueryRow(ctx, ` + SELECT c.id::text, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = $2 + WHERE c.id = $1`, chatID, userID).Scan(&c.ID, &c.Title, &c.CreatedAt) + if err != nil { + return Chat{}, mapDB(err) + } + c.CreatedAt = c.CreatedAt.UTC() + ids, err := p.memberIDs(ctx, p.pool, []string{c.ID}) + if err != nil { + return Chat{}, err + } + c.MemberIDs = ids[c.ID] + if c.MemberIDs == nil { + c.MemberIDs = []string{} + } + return c, nil +} + +func (p *PG) ListChats(ctx context.Context, userID string, cur *Cursor, limit int) ([]Chat, bool, error) { + hasCursor := cur != nil + at := time.Time{} + var id any + if hasCursor { + at = cur.At + id = cur.ID + } + rows, err := p.pool.Query(ctx, ` + SELECT c.id::text, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = $1 + WHERE NOT $2::bool OR (c.created_at, c.id) > ($3::timestamptz, $4::uuid) + ORDER BY c.created_at, c.id + LIMIT $5`, userID, hasCursor, at, id, limit+1) + if err != nil { + return nil, false, mapDB(err) + } + defer rows.Close() + items := make([]Chat, 0) + ids := make([]string, 0) + for rows.Next() { + var c Chat + if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil { + return nil, false, mapDB(err) + } + c.CreatedAt = c.CreatedAt.UTC() + items = append(items, c) + ids = append(ids, c.ID) + } + if err := rows.Err(); err != nil { + return nil, false, mapDB(err) + } + hasMore := len(items) > limit + if hasMore { + items = items[:limit] + ids = ids[:limit] + } + members, err := p.memberIDs(ctx, p.pool, ids) + if err != nil { + return nil, false, err + } + for i := range items { + items[i].MemberIDs = members[items[i].ID] + if items[i].MemberIDs == nil { + items[i].MemberIDs = []string{} + } + } + return items, hasMore, nil +} + +func (p *PG) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, fingerprint string) (Message, error) { + if idemKey == "" || fingerprint == "" { + return Message{}, ErrInvalid + } + var last error + for attempt := 0; attempt < 5; attempt++ { + msg, retry, err := p.addMessageOnce(ctx, chatID, senderID, text, attachmentIDs, idemKey, fingerprint) + if !retry { + return msg, err + } + last = err + } + if last == nil { + last = ErrUnavailable + } + return Message{}, ErrUnavailable +} + +// addMessageOnce пишет сообщение и ключ одной транзакцией. +// retry=true значит, что параллельная реплика заняла тот же ключ раньше: свою вставку откатываем и читаем её результат. +func (p *PG) addMessageOnce(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, fingerprint string) (Message, bool, error) { + tx, err := p.pool.Begin(ctx) + if err != nil { + return Message{}, false, mapDB(err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var member int + if err := tx.QueryRow(ctx, ` + SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2`, chatID, senderID).Scan(&member); err != nil { + return Message{}, false, mapDB(err) + } + + var storedFP, messageID string + var expires time.Time + err = tx.QueryRow(ctx, ` + SELECT fingerprint, message_id::text, expires_at + FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND idem_key = $3 + FOR UPDATE`, senderID, chatID, idemKey).Scan(&storedFP, &messageID, &expires) + if err == nil { + if expires.After(time.Now()) { + if storedFP != fingerprint { + return Message{}, false, ErrIdempotencyMismatch + } + msg, err := scanMessage(tx.QueryRow(ctx, ` + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM messages WHERE id = $1`, messageID)) + if err != nil { + return Message{}, false, err + } + msg.AttachmentIDs, err = attachmentIDsOf(ctx, tx, msg.ID) + if err != nil { + return Message{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return Message{}, false, mapDB(err) + } + return msg, false, nil + } + if _, err := tx.Exec(ctx, ` + DELETE FROM message_idempotency + WHERE user_id = $1 AND chat_id = $2 AND idem_key = $3`, senderID, chatID, idemKey); err != nil { + return Message{}, false, mapDB(err) + } + } else if !errors.Is(err, pgx.ErrNoRows) { + return Message{}, false, mapDB(err) + } + + now := time.Now().UTC().Truncate(time.Microsecond) + m := Message{ + ID: newID(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: now, + AttachmentIDs: append([]string(nil), attachmentIDs...), + } + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + if _, err := tx.Exec(ctx, ` + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5)`, m.ID, m.ChatID, m.SenderID, m.Text, m.CreatedAt); err != nil { + return Message{}, retryable(err), mapDB(err) + } + if err := bindAttachments(ctx, tx, m.ID, chatID, m.AttachmentIDs); err != nil { + return Message{}, retryable(err), mapDB(err) + } + recorded := time.Now().UTC().Truncate(time.Microsecond) + if _, err := tx.Exec(ctx, ` + INSERT INTO message_idempotency + (user_id, chat_id, idem_key, fingerprint, message_id, created_at, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + senderID, chatID, idemKey, fingerprint, m.ID, recorded, recorded.Add(p.idemTTL)); err != nil { + return Message{}, retryable(err), mapDB(err) + } + if err := tx.Commit(ctx); err != nil { + return Message{}, retryable(err), mapDB(err) + } + return m, false, nil +} + +func retryable(err error) bool { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505", "40001", "40P01": + return true + } + } + return false +} + +func (p *PG) ListMessages(ctx context.Context, chatID, userID string, cur *Cursor, limit int) ([]Message, bool, error) { + var member int + err := p.pool.QueryRow(ctx, ` + SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2`, chatID, userID).Scan(&member) + if err != nil { + return nil, false, mapDB(err) + } + if cur != nil { + var one int + err := p.pool.QueryRow(ctx, ` + SELECT 1 FROM messages + WHERE chat_id = $1 AND id = $2 AND created_at = $3`, chatID, cur.ID, cur.At).Scan(&one) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, ErrBadCursor + } + if err != nil { + return nil, false, mapDB(err) + } + } + hasCursor := cur != nil + at := time.Time{} + var id any + if hasCursor { + at = cur.At + id = cur.ID + } + rows, err := p.pool.Query(ctx, ` + SELECT id::text, chat_id::text, sender_id::text, text, created_at + FROM messages + WHERE chat_id = $1 + AND (NOT $2::bool OR (created_at, id) > ($3::timestamptz, $4::uuid)) + ORDER BY created_at, id + LIMIT $5`, chatID, hasCursor, at, id, limit+1) + if err != nil { + return nil, false, mapDB(err) + } + defer rows.Close() + items := make([]Message, 0) + for rows.Next() { + m, err := scanMessage(rows) + if err != nil { + return nil, false, err + } + items = append(items, m) + } + if err := rows.Err(); err != nil { + return nil, false, mapDB(err) + } + hasMore := len(items) > limit + if hasMore { + items = items[:limit] + } + if err := fillAttachmentIDs(ctx, p.pool, items); err != nil { + return nil, false, err + } + return items, hasMore, nil +} + +type queryer interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) +} + +func (p *PG) memberIDs(ctx context.Context, q queryer, chatIDs []string) (map[string][]string, error) { + out := make(map[string][]string, len(chatIDs)) + if len(chatIDs) == 0 { + return out, nil + } + rows, err := q.Query(ctx, ` + SELECT chat_id::text, user_id::text + FROM chat_members + WHERE chat_id::text = ANY($1::text[]) + ORDER BY chat_id, position`, chatIDs) + if err != nil { + return nil, mapDB(err) + } + defer rows.Close() + for rows.Next() { + var chatID, userID string + if err := rows.Scan(&chatID, &userID); err != nil { + return nil, mapDB(err) + } + out[chatID] = append(out[chatID], userID) + } + if err := rows.Err(); err != nil { + return nil, mapDB(err) + } + return out, nil +} + +func scanMessage(row pgx.Row) (Message, error) { + var m Message + if err := row.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + return Message{}, mapDB(err) + } + m.CreatedAt = m.CreatedAt.UTC() + return m, nil +} + +func scanUser(row pgx.Row) (User, error) { + var u User + err := row.Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) + if err != nil { + return User{}, mapDB(err) + } + u.CreatedAt = u.CreatedAt.UTC() + return u, nil +} + +func mapDB(err error) error { + if err == nil { + return nil + } + if errors.Is(err, pgx.ErrNoRows) { + return ErrNotFound + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return ErrUnavailable + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + switch pgErr.Code { + case "23505": + return ErrConflict + case "23503", "23514", "22P02": + return ErrInvalid + case "57014", "55P03": + return ErrUnavailable + } + } + var netErr net.Error + if errors.As(err, &netErr) { + return ErrUnavailable + } + return err +} diff --git a/internal/store/postgres_test.go b/internal/store/postgres_test.go new file mode 100644 index 00000000..04ba02c0 --- /dev/null +++ b/internal/store/postgres_test.go @@ -0,0 +1,183 @@ +package store + +import ( + "context" + "errors" + "os" + "sync" + "testing" + "time" + + "messenger/internal/migrate" +) + +func openIntegration(t *testing.T) *PG { + t.Helper() + raw := os.Getenv("DATABASE_URL") + if raw == "" { + t.Skip("DATABASE_URL is not set") + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + pg, err := Open(ctx, raw, 10) + if err != nil { + t.Fatal(err) + } + t.Cleanup(pg.Close) + if err := migrate.Apply(ctx, pg.Pool()); err != nil { + t.Fatal(err) + } + return pg +} + +func TestIntegrationMigrationRerunKeepsRows(t *testing.T) { + pg := openIntegration(t) + ctx := context.Background() + name := "mig_" + newID()[:8] + if _, err := pg.CreateUser(ctx, name, "Миграция", "hash"); err != nil { + t.Fatal(err) + } + var before int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&before); err != nil { + t.Fatal(err) + } + if err := migrate.Apply(ctx, pg.Pool()); err != nil { + t.Fatal(err) + } + var after int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM users`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatalf("migration rerun changed users %d -> %d", before, after) + } +} + +func TestIntegrationUsernameRace(t *testing.T) { + pg := openIntegration(t) + ctx := context.Background() + name := "race_" + newID()[:8] + const n = 32 + var wg sync.WaitGroup + wins := make(chan struct{}, n) + fails := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, err := pg.CreateUser(ctx, name, "Гонка", "hash") + if err == nil { + wins <- struct{}{} + return + } + fails <- err + }() + } + wg.Wait() + close(wins) + close(fails) + got := 0 + for range wins { + got++ + } + if got != 1 { + t.Fatalf("winners = %d, want 1", got) + } + for err := range fails { + if !errors.Is(err, ErrConflict) { + t.Fatal(err) + } + } +} + +func TestIntegrationChatRollback(t *testing.T) { + pg := openIntegration(t) + ctx := context.Background() + u, err := pg.CreateUser(ctx, "rb_"+newID()[:8], "Откат", "hash") + if err != nil { + t.Fatal(err) + } + var before int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM chats`).Scan(&before); err != nil { + t.Fatal(err) + } + _, err = pg.CreateChat(ctx, "не должен сохраниться", []string{u.ID, "99999999-9999-4999-8999-999999999999"}, u.ID) + if !errors.Is(err, ErrInvalid) { + t.Fatalf("want invalid, got %v", err) + } + var after int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM chats`).Scan(&after); err != nil { + t.Fatal(err) + } + if before != after { + t.Fatalf("rolled-back chat stayed: %d -> %d", before, after) + } + var members int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM chat_members WHERE user_id = $1`, u.ID).Scan(&members); err != nil { + t.Fatal(err) + } + if members != 0 { + t.Fatalf("rolled-back membership stayed: %d", members) + } +} + +func TestIntegrationIdempotencyRace(t *testing.T) { + pg := openIntegration(t) + ctx := context.Background() + u, err := pg.CreateUser(ctx, "idm_"+newID()[:8], "Ключ", "hash") + if err != nil { + t.Fatal(err) + } + chat, err := pg.CreateChat(ctx, "идемпотентность", []string{u.ID}, u.ID) + if err != nil { + t.Fatal(err) + } + const n = 16 + var wg sync.WaitGroup + ids := make(chan string, n) + errs := make(chan error, n) + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + msg, err := pg.AddMessage(ctx, chat.ID, u.ID, "одно", nil, "key-1", "одно") + if err != nil { + errs <- err + return + } + ids <- msg.ID + }() + } + wg.Wait() + close(ids) + close(errs) + for err := range errs { + t.Fatal(err) + } + seen := map[string]struct{}{} + for id := range ids { + seen[id] = struct{}{} + } + if len(seen) != 1 { + t.Fatalf("concurrent key created %d messages", len(seen)) + } + if _, err := pg.AddMessage(ctx, chat.ID, u.ID, "другое", nil, "key-1", "другое"); !errors.Is(err, ErrIdempotencyMismatch) { + t.Fatalf("mismatch: %v", err) + } + var count int + if err := pg.Pool().QueryRow(ctx, `SELECT count(*) FROM messages WHERE chat_id = $1`, chat.ID).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Fatalf("messages = %d, want 1", count) + } +} + +func TestPostgresUnreachable(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, err := Open(ctx, "postgres://messenger:secret@127.0.0.1:1/messenger?sslmode=disable&connect_timeout=1", 1) + if err == nil { + t.Fatal("closed port was accepted") + } +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 00000000..33fc819b --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,415 @@ +package store + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "sync" + "time" +) + +var ( + ErrConflict = errors.New("conflict") + ErrIdempotencyMismatch = errors.New("idempotency mismatch") + ErrNotFound = errors.New("not found") + ErrInvalid = errors.New("invalid") + ErrBadCursor = errors.New("bad cursor") + ErrUnavailable = errors.New("unavailable") + ErrNotReady = errors.New("attachment not ready") +) + +// User — публичная карточка. Хеш пароля в неё не входит. +type User struct { + ID string + Username string + DisplayName string + CreatedAt time.Time +} + +// Chat — неизменяемый состав участников. +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +// Message — подтверждённое сообщение. Текст после записи не меняется. +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time + AttachmentIDs []string +} + +// Attachment — метаданные файла. ObjectKey наружу API не отдаётся. +type Attachment struct { + ID string + ChatID string + OwnerID string + Filename string + ContentType string + SizeBytes int64 + SHA256 string + ObjectKey string + Width int + Height int + Status string + CreatedAt time.Time + UpdatedAt time.Time +} + +// AttachmentRef — строка для сверки с bucket. +type AttachmentRef struct { + ID string + ObjectKey string + CreatedAt time.Time + Referenced bool +} + +// Cursor — позиция keyset (created_at, id). Пустые поля означают первую страницу. +type Cursor struct { + ID string + At time.Time +} + +// Repository — пользователи, чаты и сообщения. Сессии сюда не входят. +type Repository interface { + Ping(ctx context.Context) error + CreateUser(ctx context.Context, username, displayName, passwordHash string) (User, error) + User(ctx context.Context, id string) (User, error) + UserByUsername(ctx context.Context, username string) (User, string, error) + CreateChat(ctx context.Context, title string, memberIDs []string, caller string) (Chat, error) + ChatFor(ctx context.Context, chatID, userID string) (Chat, error) + ListChats(ctx context.Context, userID string, cur *Cursor, limit int) ([]Chat, bool, error) + // AddMessage создаёт сообщение или возвращает уже записанное для того же ключа. + // fingerprint — каноническое тело. Другой fingerprint при живом ключе даёт ErrIdempotencyMismatch. + // Ключ живёт idempotencyTTL. Область: отправитель + чат + ключ (метод POST и путь чата). + // Чужое или неизвестное вложение — ErrNotFound. Неготовое — ErrNotReady. + AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, fingerprint string) (Message, error) + ListMessages(ctx context.Context, chatID, userID string, cur *Cursor, limit int) ([]Message, bool, error) + InsertAttachment(ctx context.Context, row Attachment) (Attachment, error) + AttachmentVisible(ctx context.Context, id, userID string) (Attachment, error) + AttachmentRefs(ctx context.Context) ([]AttachmentRef, error) + DeleteUnreferencedAttachment(ctx context.Context, id string, olderThan time.Time) (bool, error) +} + +type chatRec struct { + chat Chat + members map[string]struct{} +} + +type idemRec struct { + fingerprint string + messageID string + expires time.Time +} + +// Memory сериализует записи мьютексом. Это двойник контракта для тестов без PostgreSQL. +type Memory struct { + mu sync.RWMutex + users map[string]User + byName map[string]string + hashes map[string]string + chats map[string]*chatRec + chatOrder []string + messages map[string][]Message + msgIndex map[string]map[string]int + attachments map[string]Attachment + referenced map[string]struct{} + idem map[string]idemRec + idemTTL time.Duration + last time.Time +} + +func New() *Memory { + return &Memory{ + users: make(map[string]User), + byName: make(map[string]string), + hashes: make(map[string]string), + chats: make(map[string]*chatRec), + messages: make(map[string][]Message), + msgIndex: make(map[string]map[string]int), + attachments: make(map[string]Attachment), + referenced: make(map[string]struct{}), + idem: make(map[string]idemRec), + idemTTL: 24 * time.Hour, + } +} + +// SetIdempotencyTTL меняет срок ключа. Нужен тестам; в PostgreSQL тот же смысл у IDEMPOTENCY_TTL_SECONDS. +func (s *Memory) SetIdempotencyTTL(d time.Duration) { + if d <= 0 { + return + } + s.mu.Lock() + s.idemTTL = d + s.mu.Unlock() +} + +func (s *Memory) Ping(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return ErrUnavailable + } + return nil +} + +func (s *Memory) CreateUser(ctx context.Context, username, displayName, passwordHash string) (User, error) { + if err := ctx.Err(); err != nil { + return User{}, ErrUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + if _, ok := s.byName[username]; ok { + return User{}, ErrConflict + } + u := User{ + ID: newID(), + Username: username, + DisplayName: displayName, + CreatedAt: s.nextTimeLocked(), + } + s.users[u.ID] = u + s.byName[username] = u.ID + s.hashes[u.ID] = passwordHash + return u, nil +} + +func (s *Memory) User(ctx context.Context, id string) (User, error) { + if err := ctx.Err(); err != nil { + return User{}, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + u, ok := s.users[id] + if !ok { + return User{}, ErrNotFound + } + return u, nil +} + +func (s *Memory) UserByUsername(ctx context.Context, username string) (User, string, error) { + if err := ctx.Err(); err != nil { + return User{}, "", ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + id, ok := s.byName[username] + if !ok { + return User{}, "", ErrNotFound + } + return s.users[id], s.hashes[id], nil +} + +func (s *Memory) CreateChat(ctx context.Context, title string, memberIDs []string, caller string) (Chat, error) { + if err := ctx.Err(); err != nil { + return Chat{}, ErrUnavailable + } + s.mu.Lock() + defer s.mu.Unlock() + seen := make(map[string]struct{}, len(memberIDs)) + for _, id := range memberIDs { + if _, dup := seen[id]; dup { + return Chat{}, ErrInvalid + } + seen[id] = struct{}{} + if _, ok := s.users[id]; !ok { + return Chat{}, ErrInvalid + } + } + if _, ok := seen[caller]; !ok { + return Chat{}, ErrInvalid + } + c := Chat{ + ID: newID(), + Title: title, + MemberIDs: append([]string(nil), memberIDs...), + CreatedAt: s.nextTimeLocked(), + } + s.chats[c.ID] = &chatRec{chat: c, members: seen} + s.chatOrder = append(s.chatOrder, c.ID) + s.messages[c.ID] = nil + s.msgIndex[c.ID] = make(map[string]int) + return c, nil +} + +func (s *Memory) ChatFor(ctx context.Context, chatID, userID string) (Chat, error) { + if err := ctx.Err(); err != nil { + return Chat{}, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + rec, ok := s.chats[chatID] + if !ok { + return Chat{}, ErrNotFound + } + if _, member := rec.members[userID]; !member { + return Chat{}, ErrNotFound + } + return rec.chat.clone(), nil +} + +func (s *Memory) ListChats(ctx context.Context, userID string, cur *Cursor, limit int) ([]Chat, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + filtered := make([]Chat, 0) + for _, id := range s.chatOrder { + rec := s.chats[id] + if _, ok := rec.members[userID]; ok { + filtered = append(filtered, rec.chat.clone()) + } + } + start, err := indexAfter(filtered, cur, func(c Chat) (string, time.Time) { + return c.ID, c.CreatedAt + }) + if err != nil { + return nil, false, err + } + end := start + limit + if end > len(filtered) { + end = len(filtered) + } + return append([]Chat(nil), filtered[start:end]...), end < len(filtered), nil +} + +func (s *Memory) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, idemKey, fingerprint string) (Message, error) { + if err := ctx.Err(); err != nil { + return Message{}, ErrUnavailable + } + if idemKey == "" || fingerprint == "" { + return Message{}, ErrInvalid + } + s.mu.Lock() + defer s.mu.Unlock() + rec, ok := s.chats[chatID] + if !ok { + return Message{}, ErrNotFound + } + if _, member := rec.members[senderID]; !member { + return Message{}, ErrNotFound + } + slot := senderID + "\x00" + chatID + "\x00" + idemKey + if prev, ok := s.idem[slot]; ok && prev.expires.After(time.Now()) { + if prev.fingerprint != fingerprint { + return Message{}, ErrIdempotencyMismatch + } + idx, ok := s.msgIndex[chatID][prev.messageID] + if !ok { + return Message{}, ErrNotFound + } + return s.messages[chatID][idx], nil + } + ids := append([]string(nil), attachmentIDs...) + for _, id := range ids { + row, ok := s.attachments[id] + if !ok || row.ChatID != chatID { + return Message{}, ErrNotFound + } + if row.Status != "ready" { + return Message{}, ErrNotReady + } + } + m := Message{ + ID: newID(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: s.nextTimeLocked(), + AttachmentIDs: ids, + } + if m.AttachmentIDs == nil { + m.AttachmentIDs = []string{} + } + idx := len(s.messages[chatID]) + s.messages[chatID] = append(s.messages[chatID], m) + s.msgIndex[chatID][m.ID] = idx + s.idem[slot] = idemRec{fingerprint: fingerprint, messageID: m.ID, expires: time.Now().Add(s.idemTTL)} + return m, nil +} + +func (s *Memory) ListMessages(ctx context.Context, chatID, userID string, cur *Cursor, limit int) ([]Message, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, ErrUnavailable + } + s.mu.RLock() + defer s.mu.RUnlock() + rec, ok := s.chats[chatID] + if !ok { + return nil, false, ErrNotFound + } + if _, member := rec.members[userID]; !member { + return nil, false, ErrNotFound + } + all := s.messages[chatID] + start := 0 + if cur != nil { + idx, ok := s.msgIndex[chatID][cur.ID] + if !ok || !all[idx].CreatedAt.Equal(cur.At) { + return nil, false, ErrBadCursor + } + start = idx + 1 + } + end := start + limit + if end > len(all) { + end = len(all) + } + out := append([]Message(nil), all[start:end]...) + return out, end < len(all), nil +} + +// nextTimeLocked выдаёт неубывающее время фиксации. +func (s *Memory) nextTimeLocked() time.Time { + now := time.Now().UTC().Truncate(time.Microsecond) + if !now.After(s.last) { + now = s.last.Add(time.Microsecond) + } + s.last = now + return now +} + +func indexAfter[T any](items []T, cur *Cursor, key func(T) (string, time.Time)) (int, error) { + if cur == nil { + return 0, nil + } + for i, item := range items { + id, at := key(item) + if id == cur.ID { + if !at.Equal(cur.At) { + return 0, ErrBadCursor + } + return i + 1, nil + } + } + return 0, ErrBadCursor +} + +func (c Chat) clone() Chat { + c.MemberIDs = append([]string(nil), c.MemberIDs...) + return c +} + +func newID() string { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + panic("crypto/rand: " + err.Error()) + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + var buf [36]byte + hex.Encode(buf[0:8], b[0:4]) + buf[8] = '-' + hex.Encode(buf[9:13], b[4:6]) + buf[13] = '-' + hex.Encode(buf[14:18], b[6:8]) + buf[18] = '-' + hex.Encode(buf[19:23], b[8:10]) + buf[23] = '-' + hex.Encode(buf[24:36], b[10:16]) + return string(buf[:]) +} diff --git a/messenger/main.go b/messenger/main.go new file mode 100644 index 00000000..4ab33dcb --- /dev/null +++ b/messenger/main.go @@ -0,0 +1,309 @@ +package main + +import ( + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "encoding/json" + + "messenger/internal/api" + "messenger/internal/blob" + "messenger/internal/bootstrap" + "messenger/internal/config" + "messenger/internal/migrate" + "messenger/internal/password" + "messenger/internal/reconcile" + "messenger/internal/session" + "messenger/internal/store" +) + +func main() { + if len(os.Args) > 1 { + switch os.Args[1] { + case "healthcheck": + os.Exit(runHealthcheck()) + case "migrate": + os.Exit(runMigrate()) + case "reconcile": + os.Exit(runReconcile(os.Args[2:])) + case "bootstrap-s3": + os.Exit(runBootstrap()) + case "s3-put": + os.Exit(runS3Put(os.Args[2:])) + case "s3-rm": + os.Exit(runS3Remove(os.Args[2:])) + } + } + + // Инициализация + cfg, err := config.Load() + if err != nil { + slog.Error("config", "err", err.Error()) + os.Exit(2) + } + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + root, stop := context.WithTimeout(context.Background(), 20*time.Second) + pg, err := store.Open(root, cfg.DatabaseURL, cfg.DBMaxConns) + if err != nil { + stop() + log.Error("postgres", "err", "connect failed") + os.Exit(1) + } + defer pg.Close() + pg.SetIdempotencyTTL(cfg.IdempotencyTTL) + sessions, err := session.OpenRedis(cfg.RedisURL) + if err != nil { + stop() + log.Error("redis", "err", "config failed") + os.Exit(1) + } + defer sessions.Close() + if err := sessions.Ping(root); err != nil { + stop() + log.Error("redis", "err", "ping failed") + os.Exit(1) + } + objects, err := blob.Open(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey) + if err != nil { + stop() + log.Error("s3", "err", "config failed") + os.Exit(1) + } + if err := objects.Ping(root); err != nil { + stop() + log.Error("s3", "err", "bucket unavailable") + os.Exit(1) + } + stop() + instanceID := cfg.InstanceID + if instanceID == "" { + instanceID, err = os.Hostname() + if err != nil || instanceID == "" { + log.Error("hostname") + os.Exit(1) + } + } + application := api.New(pg, sessions, password.Params{ + Time: cfg.Argon2Time, + Memory: cfg.Argon2Memory, + Threads: cfg.Argon2Threads, + }, cfg.SessionTTL, instanceID, log) + application.EnableAttachments(objects, cfg.MaxUploadBytes, cfg.MaxImagePixels) + // http сервер + srv := &http.Server{ + Addr: cfg.Addr(), + Handler: application, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 120 * time.Second, + WriteTimeout: 120 * time.Second, + IdleTimeout: 60 * time.Second, + MaxHeaderBytes: 1 << 20, + } + + // Запуск сервера + errCh := make(chan error, 1) + go func() { + log.Info("listening", + "addr", cfg.Addr(), + "env", cfg.Env, + "instance_id", instanceID, + "shutdown_timeout_seconds", int(cfg.ShutdownTimeout/time.Second), + ) + err := srv.ListenAndServe() + if err != nil && !errors.Is(err, http.ErrServerClosed) { + errCh <- err + } + close(errCh) + }() + + // Ожидание сигнала или ошибки + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM, syscall.SIGINT) + select { + case err := <-errCh: + log.Error("server", "err", err.Error()) + os.Exit(1) + case <-sig: + } + + // Плавное завершение + application.BeginShutdown() + ctx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + log.Error("shutdown timeout, closing listener", "err", err.Error()) + _ = srv.Close() + } + log.Info("shutdown complete") +} + +func runMigrate() int { + db := os.Getenv("DATABASE_URL") + if db == "" { + slog.Error("migrate", "err", "DATABASE_URL is required") + return 2 + } + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + pg, err := store.Open(ctx, db, 2) + if err != nil { + log.Error("migrate", "err", "connect failed") + return 1 + } + defer pg.Close() + if err := migrate.Apply(ctx, pg.Pool()); err != nil { + log.Error("migrate", "err", "apply failed") + return 1 + } + log.Info("migrate complete") + return 0 +} + +func runReconcile(args []string) int { + apply := false + for _, arg := range args { + if arg == "--apply" { + apply = true + continue + } + slog.Error("reconcile", "err", "usage: reconcile [--apply]") + return 2 + } + cfg, err := config.Load() + if err != nil { + slog.Error("config", "err", err.Error()) + return 2 + } + log := slog.New(slog.NewJSONHandler(os.Stderr, nil)) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + pg, err := store.Open(ctx, cfg.DatabaseURL, 2) + if err != nil { + log.Error("postgres", "err", "connect failed") + return 1 + } + defer pg.Close() + objects, err := blob.Open(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey) + if err != nil { + log.Error("s3", "err", "config failed") + return 1 + } + summary, err := reconcile.Run(ctx, pg, objects, cfg.ReconcileGrace, apply) + if err != nil { + log.Error("reconcile", "err", "run failed") + return 1 + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(summary); err != nil { + return 1 + } + return 0 +} + +func runBootstrap() int { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + err := bootstrap.Bucket(ctx, + os.Getenv("S3_ENDPOINT"), + os.Getenv("S3_REGION"), + os.Getenv("S3_BUCKET"), + os.Getenv("MINIO_ROOT_USER"), + os.Getenv("MINIO_ROOT_PASSWORD"), + os.Getenv("S3_ACCESS_KEY"), + os.Getenv("S3_SECRET_KEY"), + ) + if err != nil { + slog.Error("bootstrap-s3", "err", err.Error()) + return 1 + } + slog.Info("bootstrap-s3 complete") + return 0 +} + +func runS3Put(args []string) int { + if len(args) != 1 || args[0] == "" { + slog.Error("s3-put", "err", "usage: s3-put ") + return 2 + } + body, err := io.ReadAll(io.LimitReader(os.Stdin, 10*1024*1024+1)) + if err != nil || len(body) == 0 || len(body) > 10*1024*1024 { + slog.Error("s3-put", "err", "body") + return 2 + } + objects, err := openS3() + if err != nil { + slog.Error("s3-put", "err", "config") + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := objects.Put(ctx, args[0], "application/octet-stream", body); err != nil { + slog.Error("s3-put", "err", "put failed") + return 1 + } + return 0 +} + +func runS3Remove(args []string) int { + if len(args) != 1 || args[0] == "" { + slog.Error("s3-rm", "err", "usage: s3-rm ") + return 2 + } + objects, err := openS3() + if err != nil { + return 1 + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := objects.Delete(ctx, args[0]); err != nil { + slog.Error("s3-rm", "err", "delete failed") + return 1 + } + return 0 +} + +func openS3() (*blob.S3, error) { + cfg, err := config.Load() + if err != nil { + slog.Error("config", "err", err.Error()) + return nil, err + } + return blob.Open(cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket, cfg.S3AccessKey, cfg.S3SecretKey) +} + +func runHealthcheck() int { + port := os.Getenv("APP_PORT") + if port == "" { + port = "8080" + } + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return 1 + } + client := http.Client{Timeout: 2 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+net.JoinHostPort("127.0.0.1", port)+"/health/ready", nil) + if err != nil { + return 1 + } + resp, err := client.Do(req) + if err != nil { + return 1 + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode != http.StatusOK { + return 1 + } + return 0 +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..637f0271 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,50 @@ +-- Пользователи, чаты, состав и сообщения. PostgreSQL — источник истины. +-- Сессии здесь не хранятся: их держит Redis с AOF. + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_password_hash_present CHECK (char_length(password_hash) > 0) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Список чатов пользователя идёт по (created_at, id). +CREATE INDEX chats_list_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_nonneg CHECK (position >= 0), + CONSTRAINT chat_members_position_unique UNIQUE (chat_id, position) +); + +-- Обратный поиск: чаты конкретного пользователя. +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset-страница истории: WHERE chat_id = $1 AND (created_at, id) > cursor +-- ORDER BY created_at, id LIMIT n. Индекс покрывает фильтр и порядок, +-- поэтому страница не читает всю историю чата. +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..61b38837 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,18 @@ +-- Общая идемпотентность POST сообщения. Область: пользователь + чат + ключ. +-- Строка пишется в той же транзакции, что и сообщение, поэтому 5xx не оставляет +-- «занятый» ключ без сообщения и не оставляет сообщение без ключа. +-- Повтор с тем же текстом читает message_id. Другой текст — конфликт на уровне API. + +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + idem_key text NOT NULL, + fingerprint text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT message_idempotency_key_len CHECK (char_length(idem_key) BETWEEN 1 AND 128), + CONSTRAINT message_idempotency_fingerprint_present CHECK (char_length(fingerprint) > 0), + CONSTRAINT message_idempotency_expiry CHECK (expires_at > created_at) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..0b780f12 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,42 @@ +-- Метаданные вложения живут в PostgreSQL, байты — в приватном bucket. +-- object_key назначает сервер. filename только для показа и не входит в ключ. + +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status CHECK (status = 'ready'), + CONSTRAINT attachments_dimensions CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, position), + CONSTRAINT message_attachments_position_nonneg CHECK (position >= 0), + CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id) +); + +-- Сверка и проверка «файл уже прикреплён» не сканируют всю историю. +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 00000000..91460af0 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,7 @@ +// хранит версионируемые SQL-файлы внутри бинарника. +package migrations + +import "embed" + +//go:embed *.sql +var FS embed.FS diff --git a/scripts/check_dependencies.sh b/scripts/check_dependencies.sh new file mode 100755 index 00000000..b2a6ec74 --- /dev/null +++ b/scripts/check_dependencies.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# Отказ Redis и PostgreSQL: анонимный запрос остаётся 401, живой Bearer не становится доверенным. +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +OUT=evidence/dependencies.txt +: > "$OUT" +log() { printf '%s\n' "$1" | tee -a "$OUT"; } + +code_of() { + # $1 name, $2 expected, $3 url, optional header in $4 + if [ "$#" -eq 4 ]; then + got=$(curl -sS -o /tmp/lab2_dep_body -w '%{http_code}' -H "$4" "$3") + else + got=$(curl -sS -o /tmp/lab2_dep_body -w '%{http_code}' "$3") + fi + log "$1 $got" + if [ "$got" != "$2" ]; then + log "FAIL $1 want $2" + exit 1 + fi +} + +wait_ready() { + i=0 + while [ "$i" -lt 90 ]; do + if curl -sf -o /dev/null http://127.0.0.1:8080/health/ready; then + return 0 + fi + i=$((i + 1)) + sleep 0.5 + done + log "FAIL ready did not return" + exit 1 +} + +trap 'rm -f /tmp/lab2_user.json /tmp/lab2_login.json /tmp/lab2_dep_body' EXIT +wait_ready +user="dep$(date +%s)" +pass="Dep-check-$(date +%s)-xxxx" +reg=$(curl -sS -o /tmp/lab2_user.json -w '%{http_code}' \ + -H 'Content-Type: application/json' \ + -d "{\"username\":\"$user\",\"display_name\":\"Зависимость\",\"password\":\"$pass\"}" \ + http://127.0.0.1:8080/api/v1/users) +if [ "$reg" != "201" ]; then + log "FAIL register $reg" + exit 1 +fi +basic=$(printf '%s' "$user:$pass" | base64 -w 0) +login=$(curl -sS -o /tmp/lab2_login.json -w '%{http_code}' \ + -X POST \ + -H "Authorization: Basic $basic" \ + http://127.0.0.1:8080/api/v1/auth/sessions) +if [ "$login" != "201" ]; then + log "FAIL login $login" + exit 1 +fi +token=$(python3 -c 'import json; print(json.load(open("/tmp/lab2_login.json"))["token"])') + +log "stop redis" +docker compose --env-file .env stop redis >> "$OUT" +code_of live_redis_down 200 http://127.0.0.1:8080/health/live +code_of ready_redis_down 503 http://127.0.0.1:8080/health/ready +code_of anon_redis_down 401 http://127.0.0.1:8080/api/v1/users/me +code_of bearer_redis_down 503 http://127.0.0.1:8080/api/v1/users/me "Authorization: Bearer $token" + +log "start redis" +docker compose --env-file .env start redis >> "$OUT" +wait_ready + +log "stop postgres" +docker compose --env-file .env stop postgres >> "$OUT" +code_of live_pg_down 200 http://127.0.0.1:8080/health/live +code_of ready_pg_down 503 http://127.0.0.1:8080/health/ready +code_of anon_pg_down 401 http://127.0.0.1:8080/api/v1/users/me +code_of bearer_pg_down 503 http://127.0.0.1:8080/api/v1/users/me "Authorization: Bearer $token" + +log "start postgres" +docker compose --env-file .env start postgres >> "$OUT" +wait_ready +code_of bearer_after 200 http://127.0.0.1:8080/api/v1/users/me "Authorization: Bearer $token" +log "PASS dependencies" +rm -f /tmp/lab2_user.json /tmp/lab2_login.json /tmp/lab2_login_hdr /tmp/lab2_dep_body diff --git a/scripts/grade4.py b/scripts/grade4.py new file mode 100755 index 00000000..94661449 --- /dev/null +++ b/scripts/grade4.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Поочерёдный отказ каждой API-реплики и гонка одного Idempotency-Key.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def compose(*args): + subprocess.check_call(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT) + + +def instances(samples=40): + client = Client("http://127.0.0.1:8080", timeout=2) + found = [] + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + found = [] + seen = set() + try: + for _ in range(samples): + item = client.call("GET", "/health/live", schema="Health")["headers"]["X-Instance-Id"] + if item not in seen: + seen.add(item) + found.append(item) + if len(found) >= 2: + return found + except Failure: + pass + time.sleep(1) + raise SystemExit("expected at least two API instances, saw " + ",".join(found)) + + +def restore_api(): + compose("up", "-d", "--no-deps", "api") + ids = subprocess.check_output( + ["docker", "compose", "--env-file", ".env", "ps", "-aq", "api"], cwd=ROOT, text=True + ).split() + if ids: + subprocess.check_call(["docker", "start", *ids], cwd=ROOT) + instances() + + +def stop_and_measure(target): + client = Client("http://127.0.0.1:8080", timeout=2) + alice, _bob, _eve, chat = client.bootstrap() + end = time.monotonic() + 12 + stats = {"ok": 0, "fail": 0, "after_ok": 0, "target_after_stop": 0, "first_recovery_s": None} + started = {} + stopped = {} + + def worker(): + local = Client("http://127.0.0.1:8080", timeout=2) + while time.monotonic() < end: + key = uuid.uuid4().hex + try: + response = local.message(alice, chat, "отказ " + key, key=key) + stats["ok"] += 1 + mark = started.get("at") + if mark is not None and stats["first_recovery_s"] is None: + stats["first_recovery_s"] = round(time.monotonic() - mark, 3) + if stopped.get("at") is not None: + stats["after_ok"] += 1 + if response["headers"].get("X-Instance-Id") == target: + stats["target_after_stop"] += 1 + except Failure: + stats["fail"] += 1 + + threads = [threading.Thread(target=worker) for _ in range(4)] + for thread in threads: + thread.start() + time.sleep(2) + started["at"] = time.monotonic() + env = os.environ.copy() + env["TARGET_INSTANCE_ID"] = target + proc = subprocess.run([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env, text=True, capture_output=True) + stopped["at"] = time.monotonic() + stats["hook_exit"] = proc.returncode + stats["hook_seconds"] = round(stopped["at"] - started["at"], 3) + for thread in threads: + thread.join() + stats["target"] = target + stats["successful_rps"] = round(stats["ok"] / 12, 3) + return stats + + +def idempotency_race(): + client = Client("http://127.0.0.1:8080", timeout=5) + alice, bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + ids = [] + errors = [] + + def once(): + try: + ids.append(Client("http://127.0.0.1:8080", timeout=5).message(alice, chat, "гонка", key=key)["json"]["id"]) + except Failure as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=once) for _ in range(12)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + page = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"] + texts = [item["text"] for item in page["items"] if item["text"] == "гонка"] + return { + "parallel_requests": 12, + "distinct_ids": len(set(ids)), + "successes": len(ids), + "errors": errors, + "history_copies": len(texts), + } + + +def main(): + os.chdir(ROOT) + before = instances() + runs = [] + for target in before: + runs.append(stop_and_measure(target)) + restore_api() + race = idempotency_race() + report = {"instances_before": before, "failover": runs, "idempotency_race": race} + dest = ROOT / "evidence" / "grade4.json" + dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if any(item["hook_exit"] != 0 or item["after_ok"] < 1 or item["target_after_stop"] != 0 for item in runs): + raise SystemExit("failover measurement failed") + if race["distinct_ids"] != 1 or race["history_copies"] != 1 or race["errors"]: + raise SystemExit("idempotency race failed") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..9b5e8472 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Перезапуск API, балансировщика и всех хранилищ. Volumes не удаляются, данные не сидируются. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres redis minio api haproxy diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..d22b0ba4 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Поочерёдная замена контейнеров api. Оставшаяся реплика продолжает принимать трафик. +# Новые контейнеры собираются из текущего Dockerfile; старые заменяются по одному. +set -eu +cd "$(dirname "$0")/.." +scale="${API_REPLICAS:-2}" +docker compose --env-file .env build api +ids=$(docker compose --env-file .env ps -q api) +for id in $ids; do + docker stop -t 8 "$id" >/dev/null + docker rm "$id" >/dev/null + docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=$scale" api >/dev/null + ready=0 + i=0 + while [ "$i" -lt 40 ]; do + if curl -fsS http://127.0.0.1:8080/health/ready >/dev/null; then + ready=1 + break + fi + i=$((i + 1)) + sleep 1 + done + if [ "$ready" -ne 1 ]; then + echo "ready did not return after replacing one replica" >&2 + exit 1 + fi +done +echo "rolling update finished" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..d1c7dbbb --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,22 @@ +#!/bin/sh +# Останавливает одну API-реплику и оставляет её остановленной. +# TARGET_INSTANCE_ID — значение X-Instance-Id (hostname контейнера). +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +target=$(printf '%s' "$TARGET_INSTANCE_ID" | tr -d '[:space:]') +found="" +for id in $(docker compose --env-file .env ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + full=$(docker inspect -f '{{.Id}}' "$id") + short=$(printf '%s' "$full" | cut -c1-12) + if [ "$host" = "$target" ] || [ "$short" = "$target" ] || [ "$full" = "$target" ]; then + found=$id + break + fi +done +if [ -z "$found" ]; then + echo "API replica not found" >&2 + exit 1 +fi +docker stop -t 8 "$found" >/dev/null diff --git a/student-tests/grade45.py b/student-tests/grade45.py new file mode 100644 index 00000000..fd0b2c2a --- /dev/null +++ b/student-tests/grade45.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +"""Негативные вложения, приватный bucket, память и трек сверки. Стенд уже запущен.""" +import json +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = "http://127.0.0.1:8080" +EVIDENCE = ROOT / "evidence" + + +def req(method, path, body=None, headers=None, content_type=None): + data = body if isinstance(body, bytes) else (None if body is None else json.dumps(body).encode()) + h = dict(headers or {}) + if content_type: + h["Content-Type"] = content_type + elif data is not None and not isinstance(body, bytes): + h["Content-Type"] = "application/json" + request = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(request, timeout=60) as resp: + return resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as exc: + return exc.code, exc.headers, exc.read() + + +def user(prefix): + name = prefix + uuid.uuid4().hex[:8] + status, _, raw = req("POST", "/api/v1/users", {"username": name, "display_name": prefix, "password": "correct-horse-1"}) + assert status == 201, raw + body = json.loads(raw) + token = basic_session(name, "correct-horse-1") + return body, token + + +def basic_session(username, password): + import base64 + raw = base64.b64encode(f"{username}:{password}".encode()).decode() + status, _, body = req("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + raw}) + assert status == 201, body + return json.loads(body)["token"] + + +def auth(token): + return {"Authorization": "Bearer " + token} + + +def chat(token, title, members): + status, _, raw = req("POST", "/api/v1/chats", {"title": title, "member_ids": members}, headers=auth(token)) + assert status == 201, raw + return json.loads(raw) + + +def upload(token, chat_id, filename, body, content_type="image/png"): + return req( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + body, + headers=auth(token), + content_type=content_type, + ) + + +def psql(sql): + out = subprocess.check_output( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "sh", "-c", + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -At -c ' + json.dumps(sql)], + cwd=ROOT, text=True) + return [line for line in out.splitlines() if line.strip()] + + +def s3(cmd, key, body=b""): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", "run", "--rm", "-T", "--no-deps", "api", cmd, key], + cwd=ROOT, input=body, capture_output=True) + if proc.returncode != 0: + raise SystemExit(proc.stderr.decode()[-500:]) + + +def reconcile(apply=False, grace=5): + args = ["docker", "compose", "--env-file", ".env", "run", "--rm", "-T", "--no-deps", + "-e", f"RECONCILE_GRACE_SECONDS={grace}", "api", "reconcile"] + if apply: + args.append("--apply") + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise SystemExit(proc.stderr[-500:]) + return json.loads(proc.stdout) + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + alice, ta = user("alice") + bob, tb = user("bob") + eve, te = user("eve") + room = chat(ta, "негативные", [alice["id"], bob["id"]]) + other = chat(ta, "чужой", [alice["id"]]) + lines = [] + + def check(name, status, raw, expected): + ok = status == expected and b"internal_error" not in raw + lines.append(f"{name}: HTTP {status} expected {expected} {'PASS' if ok else 'FAIL'} body={raw[:180]!r}") + if not ok: + raise SystemExit("\n".join(lines)) + + status, _, raw = upload(ta, room["id"], "a/b.png", png) + check("slash filename", status, raw, 400) + status, _, raw = upload(ta, room["id"], "a\\b.png", png) + check("backslash filename", status, raw, 400) + status, _, raw = upload(ta, room["id"], "note.png", b"not-an-image") + check("text as png", status, raw, 415) + status, _, raw = upload(ta, room["id"], "swap.png", b"\xff\xd8\xff\xe0" + b"\x00" * 32, "image/png") + check("mime swap", status, raw, 415) + status, _, raw = upload(ta, room["id"], "cut.png", png[:80]) + check("truncated png", status, raw, 415) + status, _, raw = upload(ta, room["id"], "big.bin", b"\x00" * (10 * 1024 * 1024 + 1)) + check("over 10MiB", status, raw, 413) + before = psql("SELECT count(*) FROM attachments")[0] + status, _, raw = upload(ta, room["id"], "pixels.png", pixel_bomb()) + check("pixel bomb", status, raw, 413) + after = psql("SELECT count(*) FROM attachments")[0] + lines.append(f"rejected uploads left attachment rows {before}->{after} {'PASS' if before == after else 'FAIL'}") + + status, _, raw = upload(ta, room["id"], "ok.png", png) + check("good upload", status, raw, 201) + att = json.loads(raw) + status, _, raw = req("POST", f"/api/v1/chats/{room['id']}/messages", + {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, + headers={**auth(ta), "Idempotency-Key": uuid.uuid4().hex}) + check("unknown attachment", status, raw, 404) + status, _, foreign_raw = upload(ta, other["id"], "foreign.png", png) + check("other chat upload", status, foreign_raw, 201) + foreign = json.loads(foreign_raw) + status, _, raw = req("POST", f"/api/v1/chats/{room['id']}/messages", + {"text": "", "attachment_ids": [foreign["id"]]}, + headers={**auth(ta), "Idempotency-Key": uuid.uuid4().hex}) + check("attachment from other chat", status, raw, 404) + status, _, raw = req("GET", f"/api/v1/attachments/{att['id']}/content", headers=auth(te)) + check("outsider download", status, raw, 404) + + key = psql(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")[0] + code = subprocess.check_output( + ["curl", "-s", "-o", "/dev/null", "-w", "%{http_code}", + f"http://127.0.0.1:9000/messenger-attachments/{key}"], text=True).strip() + lines.append(f"anonymous S3 GET {code} {'PASS' if code in ('403', '401') else 'FAIL'}") + console = subprocess.run(["curl", "-s", "-o", "/dev/null", "-m", "2", "-w", "%{http_code}", "http://127.0.0.1:9001/"], + capture_output=True, text=True) + lines.append(f"console on host http_code={console.stdout.strip() or 'no-response'} err={console.returncode}") + + # 10 одновременных загрузок по 1 МиБ мусора: каждая обрезается лимитом чтения, 500 быть не должно. + junk = b"\x00" * (1 * 1024 * 1024) + codes = [] + peak = {"mem": 0} + + def one(): + status, _, _ = upload(ta, room["id"], "junk.png", junk) + codes.append(status) + + def sample(): + end = time.time() + 8 + while time.time() < end: + out = subprocess.check_output( + ["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], text=True) + for line in out.splitlines(): + if "api" not in line: + continue + mem = line.split()[1] + num = float(mem.replace("MiB", "").replace("GiB", "")) + if "GiB" in mem: + num *= 1024 + peak["mem"] = max(peak["mem"], num) + time.sleep(0.3) + + sampler = threading.Thread(target=sample) + sampler.start() + threads = [threading.Thread(target=one) for _ in range(10)] + t0 = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + sampler.join() + elapsed = time.perf_counter() - t0 + lines.append(f"10 parallel 1MiB uploads statuses={sorted(codes)} seconds={elapsed:.2f} peak_api_MiB={peak['mem']:.1f}") + if any(c == 500 for c in codes) or len(codes) != 10: + raise SystemExit("\n".join(lines)) + + # Трек 2. grace 5 секунд. + status, _, raw = upload(ta, room["id"], "used.png", png) + used = json.loads(raw) + req("POST", f"/api/v1/chats/{room['id']}/messages", + {"text": "", "attachment_ids": [used["id"]]}, + headers={**auth(ta), "Idempotency-Key": uuid.uuid4().hex}) + status, _, raw = upload(ta, room["id"], "loose.png", png) + loose = json.loads(raw) + s3("s3-put", "o/orphan-old", b"orphan-old") + time.sleep(6) + for row_id in (used["id"], loose["id"]): + object_key = psql(f"SELECT object_key FROM attachments WHERE id = '{row_id}'")[0] + s3("s3-rm", object_key) + status, _, raw = upload(ta, room["id"], "fresh.png", png) + fresh = json.loads(raw) + s3("s3-put", "o/orphan-fresh", b"orphan-fresh") + dry1 = reconcile(False) + dry2 = reconcile(False) + lines.append("dry-run-1 " + json.dumps(dry1, ensure_ascii=False)) + lines.append("dry-run-2 same=" + str(dry1["deleted_objects"] == 0 and dry2["deleted_objects"] == 0 and dry1["orphan_stale"] == dry2["orphan_stale"])) + applied = reconcile(True) + lines.append("apply " + json.dumps(applied, ensure_ascii=False)) + again = reconcile(True) + lines.append("apply-again " + json.dumps({k: again[k] for k in ("deleted_objects", "deleted_rows", "orphan_stale", "missing_stale_unreferenced")})) + status, _, _ = req("GET", f"/api/v1/attachments/{used['id']}", headers=auth(tb)) + lines.append(f"referenced metadata after apply HTTP {status}") + status, _, _ = req("GET", f"/api/v1/attachments/{used['id']}/content", headers=auth(tb)) + lines.append(f"referenced bytes missing HTTP {status}") + status, _, _ = req("GET", f"/api/v1/attachments/{loose['id']}", headers=auth(ta)) + lines.append(f"stale unreferenced metadata HTTP {status}") + status, _, content = req("GET", f"/api/v1/attachments/{fresh['id']}/content", headers=auth(tb)) + lines.append(f"fresh download HTTP {status} same_bytes={content == png}") + (EVIDENCE / "reconcile-apply.json").write_text(json.dumps({"dry": dry1, "apply": applied, "again": again}, indent=2) + "\n") + (EVIDENCE / "grade45.txt").write_text("\n".join(lines) + "\n") + print("\n".join(lines)) + + +def pixel_bomb(): + # Минимальный PNG 1×1 с подменой IHDR на 20000×2001 и пересчитанным CRC. + import struct + import zlib + sig = b"\x89PNG\r\n\x1a\n" + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", 20000, 2001, 8, 2, 0, 0, 0) + raw = b"\x00" + b"\x00\x00\x00" + return sig + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +if __name__ == "__main__": + main() diff --git a/student-tests/mkjpeg.go b/student-tests/mkjpeg.go new file mode 100644 index 00000000..4fb362c0 --- /dev/null +++ b/student-tests/mkjpeg.go @@ -0,0 +1,18 @@ +//go:build ignore + +package main + +import ( + "image" + "image/color" + "image/jpeg" + "os" +) + +func main() { + img := image.NewRGBA(image.Rect(0, 0, 8, 4)) + img.Set(0, 0, color.RGBA{R: 255, A: 255}) + if err := jpeg.Encode(os.Stdout, img, &jpeg.Options{Quality: 80}); err != nil { + os.Exit(1) + } +} diff --git a/student-tests/s3_rights.go b/student-tests/s3_rights.go new file mode 100644 index 00000000..9bced8dd --- /dev/null +++ b/student-tests/s3_rights.go @@ -0,0 +1,46 @@ +//go:build ignore + +// Проверка, что прикладной ключ видит свой bucket и не создаёт другой. +// Печатает только код ошибки, не ключи. +package main + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func main() { + client, err := minio.New("127.0.0.1:9000", &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv("S3_ACCESS_KEY"), os.Getenv("S3_SECRET_KEY"), ""), + Secure: false, + Region: os.Getenv("S3_REGION"), + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + fmt.Println("client: config error") + os.Exit(1) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + own := os.Getenv("S3_BUCKET") + ok, err := client.BucketExists(ctx, own) + fmt.Printf("BucketExists %s: ok=%v no_error=%v\n", own, ok, err == nil) + err = client.MakeBucket(ctx, "messenger-other-bucket", minio.MakeBucketOptions{Region: os.Getenv("S3_REGION")}) + code := "nil" + if err != nil { + resp := minio.ToErrorResponse(err) + code = resp.Code + if code == "" { + code = "error" + } + } + fmt.Printf("MakeBucket messenger-other-bucket: %s\n", code) + if code == "nil" || code == "" { + os.Exit(1) + } +} -- 2.54.0 From f936985d5952a895d0e8ab379ef7be76e82f97f0 Mon Sep 17 00:00:00 2001 From: idizz Date: Fri, 13 Nov 2026 19:20:37 +0300 Subject: [PATCH 2/2] readme --- infra/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 infra/README.md diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 00000000..71f64506 --- /dev/null +++ b/infra/README.md @@ -0,0 +1,18 @@ +# Инфраструктура студента + +Стенд — корневой `compose.yaml`. `compose.example.yaml` не запускается. + +| Роль | service | Образ | +| --- | --- | --- | +| PostgreSQL | postgres | postgres:16.10-bookworm | +| Redis | redis | redis:7.4.6-alpine | +| S3 | minio | alpine/minio:RELEASE.2025-10-15T17-29-55Z, digest sha256:cf23643a6cf9ce159c57643ceb88279e431262282428c9e0bf3a7ef1a97e84b4 | +| Bootstrap bucket | minio-init | тот же образ API, команда `bootstrap-s3` | +| Балансировщик | haproxy | haproxy:2.8.14 | +| API | api | сборка из Dockerfile, тег messenger-api:lab4 | + +Сервер MinIO берётся из Alpine-сборки релиза `RELEASE.2025-10-15T17-29-55Z`. У бинарника в этой сборке строка версии `DEVELOPMENT.GOGET`: в образ не передали release ldflags. Закреплены тег и digest. + +Bucket `messenger-attachments` приватный. Консоль слушает `127.0.0.1:9001` внутри контейнера и на хост не проброшена. Порт S3 на хосте — `127.0.0.1:9000`. Root нужен только `minio-init`. API использует `S3_ACCESS_KEY` с политикой `messenger-app` на один bucket. + +Повтор bootstrap: `docker compose --env-file .env up -d minio-init` после healthy MinIO. Команда создаёт bucket, политику и пользователя, если их ещё нет, и проверяет, что прикладной ключ видит bucket. -- 2.54.0