diff --git a/.env.example b/.env.example index d99d26a2..ca6e2665 100644 --- a/.env.example +++ b/.env.example @@ -7,14 +7,27 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Student implementation knobs (see REPORT.md) +SESSION_CACHE_MAX_TTL_SECONDS=60 +DB_MAX_CONNS=10 +API_CPUS=1.0 +API_MEMORY=512M +API_GOMEMLIMIT=400MiB +SHUTDOWN_DRAIN_MS=3000 +S3_TIMEOUT=15s +UPLOAD_CONCURRENCY=8 +DECODE_BUDGET_BYTES=134217728 +UPLOAD_TIMEOUT=60s +RECONCILE_GRACE=15m +RECONCILE_EVERY=10m diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b2b5733f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 + +FROM golang:1.23.4-alpine3.21 AS build +WORKDIR /src + +COPY src/go.mod src/go.sum ./ +RUN go mod download + +COPY src/ ./ +ENV CGO_ENABLED=0 +RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server + +FROM alpine:3.21.2 AS runtime +RUN addgroup -S app && adduser -S -G app app +WORKDIR /app +COPY --from=build /out/server /app/server +ARG APP_BUILD=dev +ENV APP_BUILD=$APP_BUILD +LABEL org.opencontainers.image.revision=$APP_BUILD +USER app +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1 +ENTRYPOINT ["/app/server"] diff --git a/Makefile b/Makefile index 5d86d171..d5d31eba 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,40 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# ---- student targets (lab4 solution) ---- +.PHONY: secrets up down ps restore test-student ops-checks memory track2-faults track2-cost + +secrets: + $(PYTHON) scripts/init_secrets.py + +up: secrets + docker compose up -d --build --wait + +down: + docker compose stop + +ps: + docker compose ps -a + +restore: + docker compose up -d --wait + +test-student: + $(PYTHON) student-tests/run_tests.py --base-url "$(BASE_URL)" --json evidence/student-tests.json + +ops-checks: + $(PYTHON) scripts/lab4ops.py private --output evidence/private.json + $(PYTHON) scripts/lab4ops.py failover-upload --output evidence/failover-upload.json + $(PYTHON) scripts/lab4ops.py no-local-file --output evidence/no-local-file.json + $(PYTHON) scripts/lab4ops.py bootstrap --output evidence/bootstrap.json + $(PYTHON) scripts/lab4ops.py s3-outage --output evidence/s3-outage.json + +memory: + $(PYTHON) scripts/lab4ops.py memory --output evidence/memory.json + +track2-faults: + $(PYTHON) scripts/track2/run_track2.py faults --output evidence/track2/faults.json + +track2-cost: + $(PYTHON) scripts/track2/run_track2.py cost --output evidence/track2/cost.json diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..969ae7d4 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,183 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кораблев Петр, БВТ2304 +- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 2 «Согласованность двух хранилищ»**) +- Commit/PR: ветка `solution/BVT2304-korablev` +- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, net/http, pgx/v5 5.7.2, go-redis/v9 9.7.0, golang.org/x/crypto (Argon2id), golang.org/x/sync 0.10.0 (semaphore), google/uuid 1.6.0, **minio-go/v7 7.0.82** (S3-клиент, работает с любым S3-совместимым сервером), стандартные `image/png` и `image/jpeg`. Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, **SeaweedFS 3.80** (`chrislusf/seaweedfs:3.80`), golang:1.23.4-alpine3.21 → alpine:3.21.2; для проверок вне API — python:3.12.8-alpine3.21. Документация: AWS S3 (uploads, SigV4, ListObjectsV2), SeaweedFS wiki (S3 API, `s3.config` identities), Go `image.DecodeConfig`. Помощь ИИ использовалась при написании отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже запускаются через `python`). На Windows тесты запускать с `PYTHONUTF8=1` (контракт в UTF-8). + +```sh +python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait # = make up +make check && make test # smoke через LB http://localhost:8080 +make test-student # собственные тесты (18 кейсов) +make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd +docker compose up -d --no-recreate --wait api # вернуть остановленную реплику +make ops-checks memory track2-faults track2-cost # уровень 4 и трек 2 (нужен Docker) +``` + +- `init_secrets.py` генерирует случайные пароли PostgreSQL/Valkey и **две S3-идентичности**: `bootstrap-admin` (только для `s3-init`) и `messenger-api` (права `Read/Write/List` только на bucket `messenger-attachments`), а также `secrets/s3_identities.json` — конфиг SeaweedFS. Всё в `secrets/` (в `.gitignore` и `.dockerignore`), контейнерам передаётся как Compose secrets `/run/secrets/*`. +- Порядок старта задан `depends_on`: postgres/valkey/s3 healthy → `migrate` (миграции под advisory lock) → `s3-init` (идемпотентное создание bucket) → `api` ×2 → `lb`; `reconciler` стартует после `s3-init`. Готовность — `--wait`/`/health/ready`. +- Безопасная остановка: `docker compose stop` (`make down`), данные в volumes `pgdata`, `s3data` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (удаляет volumes — необратимо). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client → HAProxy (lb, 127.0.0.1:8080) → api ×2 (Go, read-only rootfs) + ├─ PostgreSQL: users, chats, messages, sessions, idempotency, + │ attachments (метаданные), message_attachments + ├─ Valkey: кэш сессий (из lab3) + └─ SeaweedFS S3 (bucket messenger-attachments, приватный): байты +reconciler (тот же образ, `reconcile --apply --every 10m`) — сверка БД ↔ S3 +``` + +**Перенесено из lab3** без изменения поведения: пользователи, Basic → Bearer-сессии (Argon2id, Valkey-кэш), чаты, keyset-пагинация, обязательный `Idempotency-Key`, `X-Instance-Id`, HAProxy с health-check, хуки restart/stop-one. **Изменено:** убран экспериментальный путь `MESSAGE_WRITE_PATH=tx` трека 1 lab3 (остался один путь — one-shot CTE), лимит памяти API 256M → 512M, таймауты HAProxy client/server 30s/10s → 75s (загрузка до 10 MiB). + +**Выбор S3-сервера.** Планировался MinIO, но на 2026-10 образы `minio/minio` и `minio/mc` недоступны ни в Docker Hub, ни в quay.io (pull access denied / 401). Выбран SeaweedFS 3.80: одна команда `weed server -s3` (master+volume+filer+S3), идентичности с правами на конкретный bucket через `-s3.config`, без анонимной идентичности — значит bucket приватный. S3-порт 8333 не публикуется, консоли master (9333) и filer (8888) тоже: сервис только в сети `data`. + +**Модель данных** (`src/internal/db/migrations/0003_attachments.sql`): +- `attachments`: `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key` (UNIQUE, генерирует сервер: `attachments//`), `status ∈ {pending, ready}`, `missing_at`, `created_at`, `updated_at`. FK `(chat_id, owner_id) → chat_members`, CHECK-и на размер ≤ 10 MiB, пиксели ≤ 40M, формат sha256. +- `message_attachments(message_id, chat_id, attachment_id, position)` с составным FK `(chat_id, attachment_id) → attachments(chat_id, id)`: **сама БД** не даёт привязать к сообщению файл другого чата. +- `messages.text` теперь может быть пустым (CHECK 0..4000); правило «пустой текст только при ≥1 вложении» проверяет API. + +**Загрузка** `POST /api/v1/chats/{id}/attachments?filename=…` (`src/internal/api/attachments.go`): +1. Аутентификация → членство в чате (посторонний получает 404 до чтения тела) → `filename` (1..255 code points, без `/`, `\`, NUL, управляющих и bidi-override символов, не `.`/`..`) → `Content-Type` ∈ {image/png, image/jpeg} иначе 415 → `Content-Length > 10 MiB` → 413 сразу. +2. Слот загрузки (семафор `UPLOAD_CONCURRENCY=8` на реплику), тело читается через `http.MaxBytesReader(10 MiB)` → 413 при превышении. Тело буферизуется в памяти (≤ 10 MiB на слот), значит память одной реплики под загрузки ограничена 8 × 10 MiB. +3. Формат определяется по magic bytes и сравнивается с заявленным (подмена → 415); `image.DecodeConfig` читает только заголовок: >40M пикселей → 413 **до** декодирования; затем полное декодирование под общим бюджетом памяти (`DECODE_BUDGET_BYTES=128 MiB`, взвешенный семафор по оценке `пиксели × байт/пиксель`) — повреждённые данные → 415. +4. Транзакционная граница: **(a)** `INSERT attachments status='pending'` → **(b)** S3 `PUT` (Content-MD5) → **(c)** `UPDATE … SET status='ready' WHERE status='pending'` → `201 ready`. 201 возвращается только когда и байты, и метаданные зафиксированы. +5. Ошибка на (b) или (c) → 503 и фоновая компенсация, которая начинается со строки: `DELETE … WHERE status='pending'`, и только если строка удалена — удаление объекта. Если `UPDATE → ready` на самом деле закоммитился, а ответ БД потерялся, строка уже ready, и объект не трогаем (иначе получилось бы ready-вложение без байтов). Если недоступна БД, остаётся stale pending; если недоступен S3, может остаться объект без строки. Оба случая убирает reconciler после grace. + +**Скачивание** `GET /attachments/{id}/content`: только участники чата (иначе 404), только `status='ready'`; `Stat` до первого байта ответа (отсутствие объекта → 404, недоступность S3 → 503), затем поток из S3 в ответ, без redirect; заголовки `Content-Type` (истинный), `Content-Length`, `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `ETag: ""`. `variant=thumbnail|crop|optimized` → 404 (вариантов в lab4 нет), неизвестный → 400. + +**Сообщение** с `attachment_ids`: 0..10 уникальных UUID (иначе 400). Проверка «все ready и в этом чате», запись idempotency, сообщения и связей — **одним SQL-оператором** (CTE), как в lab3. Чужое, неизвестное или из другого чата → 404, ничего не сохраняется. Отпечаток идемпотентности считается по `{text, attachment_ids}` с нормализацией «нет поля» = `[]`. + +**Готовность и отказы.** `/health/ready` зависит только от PostgreSQL. Недоступность S3 ломает только эндпоинты вложений (503), а текстовый мессенджер продолжает работать: если бы ready зависел от S3, LB снял бы обе реплики и упал бы весь API. SDK S3 настроен на быстрый отказ: `MaxRetry=1`, dial-таймаут 2 с, `ResponseHeaderTimeout` 10 с, операции под `S3_TIMEOUT=15s`. + +**Соответствие `.env.example`** — все переменные реально используются в `compose.yaml`: `S3_ENDPOINT=http://s3:8333`, `S3_REGION`, `S3_BUCKET`, `S3_*_FILE` (secrets), `MAX_UPLOAD_BYTES` (не больше 10 MiB, иначе старт с ошибкой), `MAX_IMAGE_PIXELS`, плюс `UPLOAD_CONCURRENCY`, `DECODE_BUDGET_BYTES`, `UPLOAD_TIMEOUT`, `S3_TIMEOUT`, `API_MEMORY`, `API_GOMEMLIMIT`, `RECONCILE_GRACE`, `RECONCILE_EVERY`. + +| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — | +| API ×2 | api | mtusi-messenger:lab4 (сборка) | `/health/ready` | нет (read-only rootfs, tmpfs /tmp 1m) | app, data | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key | +| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password | +| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth | +| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data, не публикуется | s3_identities.json | +| Bootstrap | migrate, s3-init | mtusi-messenger:lab4 | one-shot | — | data | database_url / s3_admin_* | +| Сверка | reconciler | mtusi-messenger:lab4 | — | — | data | database_url, s3 api-ключи | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Окружение всех замеров: Windows 11, Docker Desktop 29.7.2 / Compose 5.5.0, WSL2 kernel 6.18; лимиты: API 1 CPU / 512M ×2, S3 1 CPU / 1G, PostgreSQL 2 CPU / 1G, reconciler 0.5 CPU / 128M. Дата — 2026-10-29. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Уровень 3 + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke | `make check`, `make test` | PASS: lab4 public API scenario | evidence/check-smoke.txt | +| Возможности lab3 сохранены | `make test-student` (8 кейсов lab3) | no-sticky, идемпотентность через 2 реплики, logout на всех репликах — PASS | evidence/student-tests.txt | +| S3 в общем стенде, байты в приватном bucket | `lab4ops.py private` | аноним GET/LIST/CreateBucket → 403; api-ключ GET → 200, sha256 совпадает; порт S3 не опубликован | evidence/private.json | +| Метаданные в PostgreSQL, не байты в БД/на диске API | `lab4ops.py no-local-file` | у API read-only rootfs (`touch` → Read-only file system), монтируются только secrets, новых файлов 0 | evidence/no-local-file.json | +| 201 ready, побайтный download, ACL | smoke + `t_upload_formats`, `t_attachment_acl` | PNG и JPEG: sha256/размер/Content-Type совпадают; посторонний получает 404 на метаданные/content/upload, без токена 401 | evidence/student-tests.txt | +| Реальные PNG/JPEG ≤10 MiB, ≤40M px, MIME по содержимому, опасные filename | `t_size_limit`, `t_bomb`, `t_corrupt_and_spoof`, `t_filenames` | PNG 10 420 335 B → 201; 10 MiB+1 → 413; заголовок 20000×20000 → 413 за 32 мс; 9 вариантов подмены/повреждения → 415; 12 опасных имён → 400 | evidence/student-tests.txt | +| Upload → стоп обслужившей реплики → download через другую | `lab4ops.py failover-upload` | загрузка через `api-2`, `docker stop api-2`, 10/10 скачиваний через `api-1` с верным sha256 | evidence/failover-upload.json | +| Failover (публичный тест) | `make test-failover` для каждой реплики | остановлена api-2: PASS, 123 записи, 0 ошибок, восстановление за 4.1 с; остановлена api-1: PASS, 116 записей, 0 ошибок, 4.1 с; файл цел в обоих случаях | evidence/failover.txt | +| Restart всего стенда с volumes | `make test-persistence` (`restart.cmd`) | PASS; postgres/valkey/s3/api перезапущены (uptime до/после в файле), файл и сессия сохранились | evidence/persistence.txt | + +### Уровень 4 + +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Тесты: большой файл, повреждённое изображение, подмена MIME, чужой attachment_id, файл другого чата; нет 500 | `make test-student` (10 кейсов lab4) | 18/18 PASS; в логах API за все прогоны нет ни одного 500 | evidence/student-tests.txt | +| Нет вечных неучтённых объектов: компенсация + сверка с grace | pending-intent + компенсация + `reconciler` (grace 15 мин, каждые 10 мин) | см. трек 2; реальный случай: после падения S3 reconciler нашёл 19 stale pending из прерванных загрузок и удалил их | evidence/track2/faults.json | +| Отказ S3 | `lab4ops.py s3-outage` | upload 503 за 2.0 с, download 503 за 2.0 с, метаданные 200, текстовое сообщение 201, ready 200; после старта S3 загрузки сразу работают, старый файл цел; pending-строк не осталось (компенсация удалила строку) | evidence/s3-outage.json | +| Ограниченная RAM, ≥10 одновременных загрузок | `make memory` | см. «Эксперименты»: пик API ≤ 311 MiB при 40 одновременных 10-МБ загрузках, плато уже после 20 | evidence/memory.json | +| Bootstrap идемпотентен | `lab4ops.py bootstrap` (двойной запуск `migrate` и `s3-init`) | «migrations up to date», `created:false` | evidence/bootstrap.json | +| Отдельные ограниченные S3 credentials; консоль закрыта; private bucket проверен вне API | `lab4ops.py private` | api-идентичность: CreateBucket → 403, PUT в другой bucket → 403; S3/консоли не опубликованы; аноним → 403 | evidence/private.json | + +### Трек 5.2 «Согласованность двух хранилищ» + +| Критерий | Команда | Результат | Файл | +| --- | --- | --- | --- | +| Повторяемый reconciler с dry-run | `server reconcile [--apply] [--grace] [--every]` | dry-run по умолчанию; advisory lock — одновременно идёт один проход | src/internal/reconcile/reconcile.go | +| Ошибки на обеих сторонах: обнаружение и исправление | `make track2-faults` | 5 из 5 внедрённых сбоев найдены, 7 исправлений, повторный apply → 0 | evidence/track2/faults.json | +| Свежие и используемые не удаляются | там же | свежий orphan, свежий pending и используемые U1/U2 остались целы | evidence/track2/faults.json | +| Стоимость обхода | `make track2-cost` | линейно, ~70 мкс/объект, 20k за 1.4–1.6 с | evidence/track2/cost.json | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Память API при одновременных загрузках (уровень 4) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Гипотеза:** пиковая память реплики ограничена `≈ база + UPLOAD_CONCURRENCY × 10 MiB + бюджет декодирования`, поэтому она выходит на плато и не растёт с числом клиентов. + +**Методика:** `scripts/lab4ops.py memory`. Синтетический шумовой PNG 1000×3480, 10 420 335 B (почти лимит). Перед каждым прогоном `docker compose restart api`, одна прогревочная загрузка, затем N одновременных загрузок (барьер) через LB. Пик читается из cgroup v2 `memory.peak` каждой реплики. По 2 повтора. + +| Одновременно | Успешно | Wall, с | p50 / max, мс | Пик API, MiB (max из 2 реплик) | +| --- | --- | --- | --- | --- | +| 1 | 1/1, 1/1 | 0.35 | 337–344 | 64.4 / 66.4 | +| 10 | 10/10, 10/10 | 3.1–4.1 | 2844–3830 / 2882–3945 | 148.2 / 151.7 | +| 20 | 20/20, 20/20 | 6.4–7.1 | 5476–6097 / 6062–6830 | 286.0 / 263.7 | +| 40 | 40/40, 40/40 | 11.8–12.6 | 8542–9590 / 11156–12044 | 264.5 / 311.1 | + +**Вывод:** память выходит на плато ~250–310 MiB между 20 и 40 клиентами (слоты по 8 на реплику). Лишние клиенты ждут слота, это видно по росту latency, а не памяти. Это ниже лимита 512M и GOMEMLIMIT 400MiB. **Ограничения:** буфер одного тела ≤10 MiB в RAM — сознательное упрощение вместо потокового парсера PNG. 40M-пиксельный RGBA PNG при декодировании занимает ~160 MB и упирается в бюджет 128 MiB (декодируется один, остальные ждут). Запуск на одном ноутбуке, клиент и стенд делят CPU. + +**Найденный отказ:** первый прогон шёл с лимитом S3 512M, и на 20 параллельных PUT процесс SeaweedFS перезапустился (RestartCount 1, логов завершения нет — kill по памяти). 18 из 20 загрузок получили корректный 503, их pending-строки остались (тогдашняя версия компенсации сначала удаляла объект и при недоступном S3 сохраняла строку) и затем были найдены reconciler как 19 stale pending, вместе с одной от теста отказа S3. Лимит S3 поднят до 1G: в повторе 0 рестартов при `memory.peak` ≈ 1 GiB. Этот пик включает page cache записываемых файлов, который ядро может вытеснить. + +### Трек 2: внедрение ошибок (grace = 1 мин для эксперимента, 15 мин в стенде) + +Методика `scripts/track2/run_track2.py faults`. Периодический reconciler на время теста останавливается. Сбои вносятся напрямую: в S3 через `scripts/track2/s3tool.py` (SigV4 на stdlib, запускается в одноразовом контейнере в сети `data`), в БД через `psql`. + +| Сбой (сторона) | Ожидание | dry-run | apply | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| F1 объект без строки (S3) | удалить после grace | orphan_object | удалён | +| F2 потеряна строка неиспользуемого U3 (БД) | объект стал сиротой → удалить | orphan_object | удалён | +| F3 потерян объект используемого U2 (S3) | **метаданные не трогать**, пометить | missing_object | `missing_at` выставлен; GET метаданных 200, content 404, сообщение по-прежнему ссылается на U2 | +| F4 падение между PUT и ready (pending + объект, 1 ч) | удалить строку, затем объект | stale_pending | удалены оба | +| F5 падение до PUT (pending без объекта, 1 ч) | удалить строку | stale_pending | удалена | +| F7 загрузка «в процессе» (свежие pending + объект) | не трогать | не найдена (row известен) | цела | +| F8 свежий объект без строки | не трогать | `skip: younger than grace` | цел | +| U1 используемый, исправный | не трогать | — | скачивается, sha256 верный | + +- **Dry-run ничего не меняет:** счётчики S3-объектов, строк, pending и missing до и после совпадают (`dry_run_changed_nothing: true`). +- **Первый apply:** 7 исправлений, 0 ошибок. +- **Повторный apply:** 0 исправлений, только `skip` свежего F8 — проход идемпотентен. +- **Восстановление:** после возврата объекта U2 «из бэкапа» (те же байты) apply выдаёт `recovered=1`, снимает `missing_at`, и скачивание отдаёт исходный sha256. +- **Сторонние находки:** dry-run отдельно учитывает находки вне чата эксперимента (`other_findings_outside_experiment`) — это остатки предыдущих прогонов. + +**Почему это безопасно.** Порядок «сначала LIST S3, потом скан БД» плюс «pending-строка пишется до PUT» означают, что у любого объекта из листинга строка уже есть. Перед каждым разрушающим действием есть повторная проверка: сирота — `ObjectKeyKnown` в БД, missing — `StatObject`, pending — `DELETE … WHERE status='pending'`. Удаляются только объекты без строк и pending-строки; ни одна из них не может быть в `message_attachments`, потому что привязать можно только ready. Используемые метаданные никогда не удаляются. Grace 15 мин намного больше бюджета загрузки (60 с). Если `UPDATE → ready` проигрывает гонку с reconciler, загрузка отвечает 503, а не 201. + +### Трек 2: стоимость обхода + +`scripts/track2/run_track2.py cost`: в БД добавляются N ready-строк (SQL `generate_series`), в S3 — N объектов по 100 B. Каждый размер замеряется трижды в dry-run, после замеров bench-данные удаляются. + +| Объектов | list S3, мс | скан БД, мс | всего, мс | мкс/объект | пик памяти reconciler, MiB | находок | +| --- | --- | --- | --- | --- | --- | --- | +| 1 229 | 52 | 6–9 | 59–61 | 48–50 | 10.1–10.9 | 0 | +| 5 229 | 258–345 | 62–70 | 326–408 | 62–78 | 15.8–17.2 | 0 | +| 20 229 | 1321–1537 | 41–95 | 1404–1618 | 69–80 | 30.8–31.2 | 0 | + +**Вывод:** стоимость линейна, примерно 70 мкс и 1 КБ RAM на объект. Больше 90% времени занимает постраничный LIST в S3 (по 1000 ключей). На 20k объектов ложных срабатываний 0. При частоте раз в 10 минут нагрузка ничтожна. **Ограничение:** все ключи держатся в памяти, на 10M объектов это ~1 ГБ. Дальше нужен обход по префиксам `attachments//` или потоковое слияние двух отсортированных списков: ключи S3 и `ORDER BY object_key` в БД. + +### Сводка отказов + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| --- | --- | --- | --- | +| `docker stop` обслужившей API-реплики | LB уводит трафик | первая успешная запись через ~4 с, 0 дублей; файл скачивается через другую | без потерь | +| Restart postgres+valkey+s3+api | данные, сессия и файл живы | PASS, тот же токен работает | без потерь | +| Остановка S3 | 503 только на вложениях | upload/download 503 за 2 с, текст и ready работают | после старта S3 сразу; pending-строк не осталось | +| SeaweedFS убит по памяти (лимит 512M) | 503, без 201 на несохранённое | 18× 503, 2× 201 | 19 pending-строк убраны reconciler; лимит поднят до 1G | +| Внедрённые orphan/missing/pending | найти, исправить безопасно | 5/5 найдены, повторный apply = no-op | missing-байты невосстановимы без бэкапа: метаданные помечены и сохранены | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host, по одному PostgreSQL, S3-узлу и LB: выдерживается отказ одной API-реплики и рестарт стенда, но не потеря хоста или диска S3. В SeaweedFS нет репликации томов. +- Тело загрузки буферизуется в RAM до 10 MiB на слот. Это ограничено и измерено, но не полностью потоковый режим. +- Полное декодирование нужно для отсева повреждённых файлов и стоит CPU: 40 одновременных загрузок 10 МБ занимают ~12 с на 2×1 CPU. +- Объект, у которого обнаружен `missing`, невосстановим без бэкапа. Reconciler только помечает его, и API отдаёт 404 на content. +- Reconciler хранит все ключи в памяти (ограничение указано выше). `size_mismatch` только сообщается, автоматически не исправляется. +- Цель failover-теста выбирается по `X-Instance-Id` обслужившего ответа, а HAProxy раздаёт запросы по кругу, поэтому без вмешательства тест детерминированно останавливал api-2. Для проверки api-1 перед запуском был сделан один лишний запрос через LB (`curl http://localhost:8080/health/live`), сдвигающий очередь; так получен прогон с остановкой api-1. +- CI-job со сборкой стенда и `make test` не добавлен: workflow skeleton проверяет только контракт. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..5e5b3c39 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,248 @@ +name: mtusi-lab4 + +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000} + DB_MAX_CONNS: ${DB_MAX_CONNS:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT: ${S3_TIMEOUT:-15s} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8} + DECODE_BUDGET_BYTES: ${DECODE_BUDGET_BYTES:-134217728} + UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s} + GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB} + +services: + postgres: + image: postgres:17.2-alpine3.21 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/pg_password + command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] + secrets: [pg_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [data] + shm_size: 256mb + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"} + stop_grace_period: 30s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.0.2-alpine3.21 + command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"] + volumes: + - ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro + secrets: [valkey_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: unless-stopped + + # SeaweedFS (master + volume + filer + S3 gateway in one process). Only the S3 + # port is used, and only inside the data network: nothing is published. + s3: + image: chrislusf/seaweedfs:3.80 + command: + - server + - -dir=/data + - -s3 + - -s3.port=8333 + - -s3.config=/run/secrets/s3_identities + - -master.volumeSizeLimitMB=64 + - -volume.max=100 + secrets: [s3_identities] + volumes: + - s3data:/data + networks: [data] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] + interval: 2s + timeout: 3s + start_period: 60s + retries: 60 + deploy: + resources: + # 512M was OOM-killed under 20 parallel 10 MiB PUTs (evidence/memory.json) + limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"} + stop_grace_period: 20s + restart: unless-stopped + + # Idempotent bucket bootstrap with the admin identity; the API never sees it. + s3-init: + image: mtusi-messenger:lab4 + pull_policy: never + command: ["s3-init"] + healthcheck: + disable: true + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: [s3_admin_access_key_id, s3_admin_secret_access_key] + networks: [data] + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.5", memory: 64M} + restart: "no" + + migrate: + build: + context: . + args: + APP_BUILD: ${APP_BUILD:-dev} + image: mtusi-messenger:lab4 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: "no" + + api: + image: mtusi-messenger:lab4 + pull_policy: never + environment: *api-env + # No writable filesystem: originals cannot end up on the replica's disk. + read_only: true + tmpfs: + - /tmp:size=1m + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [app, data] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + valkey: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 2s + timeout: 2s + start_period: 5s + retries: 3 + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"} + stop_grace_period: 15s + restart: unless-stopped + + # Periodic DB<->S3 reconciliation (orphan objects, stale pending uploads, + # missing objects). Never touches anything younger than RECONCILE_GRACE. + reconciler: + image: mtusi-messenger:lab4 + pull_policy: never + command: ["reconcile", "--apply", "--every", "${RECONCILE_EVERY:-10m}"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_MAX_CONNS: "2" + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + RECONCILE_GRACE: ${RECONCILE_GRACE:-15m} + healthcheck: + disable: true + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3-init: + condition: service_completed_successfully + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: unless-stopped + + lb: + image: haproxy:3.0.5-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + - "[::1]:${APP_PORT:-8080}:8080" + networks: [app] + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 2s + timeout: 2s + retries: 15 + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M} + restart: unless-stopped + +networks: + app: {} + data: {} + +volumes: + pgdata: {} + s3data: {} + +secrets: + pg_password: + file: ./secrets/pg_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_auth: + file: ./secrets/valkey_auth.conf + s3_identities: + file: ./secrets/s3_identities.json + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key diff --git a/evidence/bootstrap.json b/evidence/bootstrap.json new file mode 100644 index 00000000..73270831 --- /dev/null +++ b/evidence/bootstrap.json @@ -0,0 +1,45 @@ +{ + "command": "bootstrap", + "date_utc": "2026-10-29T10:38:50Z", + "commit": "ec540a1", + "result": [ + { + "service": "migrate", + "last_log": { + "time": "2026-10-29T10:38:51.949471511Z", + "level": "INFO", + "msg": "migrations up to date", + "total": 3 + } + }, + { + "service": "s3-init", + "last_log": { + "time": "2026-10-29T10:38:53.253676835Z", + "level": "INFO", + "msg": "bucket ready", + "bucket": "messenger-attachments", + "created": false + } + }, + { + "service": "migrate", + "last_log": { + "time": "2026-10-29T10:38:54.658943295Z", + "level": "INFO", + "msg": "migrations up to date", + "total": 3 + } + }, + { + "service": "s3-init", + "last_log": { + "time": "2026-10-29T10:38:56.030781559Z", + "level": "INFO", + "msg": "bucket ready", + "bucket": "messenger-attachments", + "created": false + } + } + ] +} diff --git a/evidence/check-smoke.txt b/evidence/check-smoke.txt new file mode 100644 index 00000000..21155968 --- /dev/null +++ b/evidence/check-smoke.txt @@ -0,0 +1,4 @@ +## make check +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +## smoke +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/failover-upload.json b/evidence/failover-upload.json new file mode 100644 index 00000000..657be042 --- /dev/null +++ b/evidence/failover-upload.json @@ -0,0 +1,54 @@ +{ + "command": "failover-upload", + "date_utc": "2026-10-29T10:38:32Z", + "commit": "ec540a1", + "result": { + "uploaded_via": "0b30fcb99d56", + "stopped_container": "mtusi-lab4-api-2", + "reads_after_stop": [ + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + }, + { + "instance": "02482efeb4f6", + "sha256_ok": true + } + ], + "seconds_observed": 9.6, + "ps_before": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 running\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running", + "ps_during": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 exited\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running" + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..574bf046 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,14 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.095} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 +mtusi-lab4-api-1 running +mtusi-lab4-api-2 exited + +## other replica (api-1 stopped) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.148} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 +mtusi-lab4-api-1 exited +mtusi-lab4-api-2 running diff --git a/evidence/memory.json b/evidence/memory.json new file mode 100644 index 00000000..da324af2 --- /dev/null +++ b/evidence/memory.json @@ -0,0 +1,190 @@ +{ + "command": "memory", + "date_utc": "2026-10-29T10:20:43Z", + "commit": "ec540a1", + "result": { + "api_limit": "512M x 2 replicas, 1 CPU each", + "gomemlimit": "400MiB", + "upload_concurrency_per_replica": 8, + "decode_budget_mib": 128, + "s3_limit": "1073741824", + "runs": [ + { + "concurrency": 1, + "repeat": 1, + "file_bytes": 10420335, + "ok": 1, + "status_503": 0, + "other": [], + "wall_s": 0.35, + "p50_ms": 337, + "max_ms": 337, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 34.4, + "mtusi-lab4-api-2": 64.4 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 29.5, + "mtusi-lab4-api-2": 54.8 + }, + "max_peak_mib": 64.4, + "s3_peak_mib_since_start": 267.1, + "s3_restarted_during_run": false + }, + { + "concurrency": 1, + "repeat": 2, + "file_bytes": 10420335, + "ok": 1, + "status_503": 0, + "other": [], + "wall_s": 0.36, + "p50_ms": 344, + "max_ms": 344, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 64.5, + "mtusi-lab4-api-2": 29.9 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 54.5, + "mtusi-lab4-api-2": 29.1 + }, + "max_peak_mib": 64.5, + "s3_peak_mib_since_start": 312.4, + "s3_restarted_during_run": false + }, + { + "concurrency": 10, + "repeat": 1, + "file_bytes": 10420335, + "ok": 10, + "status_503": 0, + "other": [], + "wall_s": 4.1, + "p50_ms": 3830, + "max_ms": 3945, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 148.2, + "mtusi-lab4-api-2": 144.4 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 54.8, + "mtusi-lab4-api-2": 30.1 + }, + "max_peak_mib": 148.2, + "s3_peak_mib_since_start": 960.0, + "s3_restarted_during_run": false + }, + { + "concurrency": 10, + "repeat": 2, + "file_bytes": 10420335, + "ok": 10, + "status_503": 0, + "other": [], + "wall_s": 3.05, + "p50_ms": 2844, + "max_ms": 2882, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 143.0, + "mtusi-lab4-api-2": 151.7 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 30.3, + "mtusi-lab4-api-2": 53.4 + }, + "max_peak_mib": 151.7, + "s3_peak_mib_since_start": 1025.2, + "s3_restarted_during_run": false + }, + { + "concurrency": 20, + "repeat": 1, + "file_bytes": 10420335, + "ok": 20, + "status_503": 0, + "other": [], + "wall_s": 6.35, + "p50_ms": 5476, + "max_ms": 6062, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 286.0, + "mtusi-lab4-api-2": 236.4 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 54.8, + "mtusi-lab4-api-2": 29.5 + }, + "max_peak_mib": 286.0, + "s3_peak_mib_since_start": 1025.2, + "s3_restarted_during_run": false + }, + { + "concurrency": 20, + "repeat": 2, + "file_bytes": 10420335, + "ok": 20, + "status_503": 0, + "other": [], + "wall_s": 7.12, + "p50_ms": 6097, + "max_ms": 6830, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 245.4, + "mtusi-lab4-api-2": 263.7 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 30.2, + "mtusi-lab4-api-2": 55.1 + }, + "max_peak_mib": 263.7, + "s3_peak_mib_since_start": 1025.2, + "s3_restarted_during_run": false + }, + { + "concurrency": 40, + "repeat": 1, + "file_bytes": 10420335, + "ok": 40, + "status_503": 0, + "other": [], + "wall_s": 12.64, + "p50_ms": 9590, + "max_ms": 12044, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 264.5, + "mtusi-lab4-api-2": 248.3 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 53.9, + "mtusi-lab4-api-2": 29.9 + }, + "max_peak_mib": 264.5, + "s3_peak_mib_since_start": 1030.8, + "s3_restarted_during_run": false + }, + { + "concurrency": 40, + "repeat": 2, + "file_bytes": 10420335, + "ok": 40, + "status_503": 0, + "other": [], + "wall_s": 11.79, + "p50_ms": 8542, + "max_ms": 11156, + "peak_mib_per_replica": { + "mtusi-lab4-api-1": 260.5, + "mtusi-lab4-api-2": 311.1 + }, + "baseline_current_mib": { + "mtusi-lab4-api-1": 29.5, + "mtusi-lab4-api-2": 54.5 + }, + "max_peak_mib": 311.1, + "s3_peak_mib_since_start": 1030.8, + "s3_restarted_during_run": false + } + ] + } +} diff --git a/evidence/no-local-file.json b/evidence/no-local-file.json new file mode 100644 index 00000000..c37a3d96 --- /dev/null +++ b/evidence/no-local-file.json @@ -0,0 +1,29 @@ +{ + "command": "no-local-file", + "date_utc": "2026-10-29T10:38:48Z", + "commit": "ec540a1", + "result": { + "mtusi-lab4-api-1": { + "readonly_rootfs": true, + "write_probe": "touch: /app/probe: Read-only file system", + "mounts": [ + "/run/secrets/s3_access_key_id", + "/run/secrets/s3_secret_access_key", + "/run/secrets/database_url", + "/run/secrets/cache_url" + ], + "files_written_since_start": [] + }, + "mtusi-lab4-api-2": { + "readonly_rootfs": true, + "write_probe": "touch: /app/probe: Read-only file system", + "mounts": [ + "/run/secrets/s3_access_key_id", + "/run/secrets/s3_secret_access_key", + "/run/secrets/database_url", + "/run/secrets/cache_url" + ], + "files_written_since_start": [] + } + } +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..eaecb914 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,21 @@ +api running Up About a minute (healthy) +api running Up About a minute (healthy) +lb running Up 11 minutes (healthy) +migrate exited Exited (0) About a minute ago +postgres running Up 11 minutes (healthy) +reconciler running Up About a minute +s3 running Up 38 seconds (healthy) +s3-init exited Exited (0) About a minute ago +valkey running Up 11 minutes (healthy) +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 +api running Up 7 seconds (healthy) +api running Up 7 seconds (healthy) +lb running Up 1 second (health: starting) +migrate exited Exited (0) 8 seconds ago +postgres running Up 46 seconds (healthy) +reconciler running Up 7 seconds +s3 running Up 36 seconds (healthy) +s3-init exited Exited (0) 8 seconds ago +valkey running Up 46 seconds (healthy) diff --git a/evidence/private.json b/evidence/private.json new file mode 100644 index 00000000..8fcfc59b --- /dev/null +++ b/evidence/private.json @@ -0,0 +1,19 @@ +{ + "command": "private", + "date_utc": "2026-10-29T10:38:17Z", + "commit": "ec540a1", + "result": { + "object_key_shape": "attachments// (server generated)", + "key_contains_filename": false, + "anonymous_get": 403, + "anonymous_list": "denied (HTTP 403)", + "api_identity_get": 200, + "api_identity_get_sha256_matches": true, + "api_identity_create_bucket": 403, + "anonymous_create_bucket": 403, + "published_s3_ports": "invalid IP:0", + "outsider_via_api_metadata": 404, + "outsider_via_api_content": 404, + "api_identity_put_other_bucket": "{\"cmd\": \"put\", \"as\": \"api\", \"status\": 403}" + } +} diff --git a/evidence/s3-outage.json b/evidence/s3-outage.json new file mode 100644 index 00000000..219a24a8 --- /dev/null +++ b/evidence/s3-outage.json @@ -0,0 +1,17 @@ +{ + "command": "s3-outage", + "date_utc": "2026-10-29T11:45:59Z", + "commit": "ec540a1", + "result": { + "upload_status": 503, + "upload_ms": 2048, + "download_status": 503, + "download_ms": 2018, + "metadata_status": 200, + "text_message_status": 201, + "ready_status": 200, + "pending_rows_left": 0, + "upload_recovered_after_s3_healthy_s": 0.1, + "old_file_intact": true + } +} diff --git a/evidence/student-tests.json b/evidence/student-tests.json new file mode 100644 index 00000000..66ab1646 --- /dev/null +++ b/evidence/student-tests.json @@ -0,0 +1,92 @@ +[ + { + "test": "one session served by >=2 replicas without re-login (no stickiness)", + "result": "PASS", + "detail": "distribution [15, 15]" + }, + { + "test": "X-Instance-Id is present on errors and health too", + "result": "PASS", + "detail": null + }, + { + "test": "Idempotency-Key missing or malformed -> 400, nothing stored", + "result": "PASS", + "detail": null + }, + { + "test": "concurrent replay of one key through different replicas -> exactly one message", + "result": "PASS", + "detail": "24 parallel requests via 2 replicas -> 1 message" + }, + { + "test": "replay is semantic: JSON key order/whitespace does not matter; other body -> 409", + "result": "PASS", + "detail": null + }, + { + "test": "key scope = user + chat: same key is independent for other user / other chat", + "result": "PASS", + "detail": null + }, + { + "test": "logout is visible on every replica immediately (shared sessions)", + "result": "PASS", + "detail": "revoked token rejected by 2 replicas" + }, + { + "test": "replay re-checks the session and chat access", + "result": "PASS", + "detail": null + }, + { + "test": "PNG and JPEG upload/download byte-exact with true Content-Type", + "result": "PASS", + "detail": "png 111817 B, jpeg 2565 B" + }, + { + "test": "size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413", + "result": "PASS", + "detail": "accepted 10420335 B, rejected 10485761 B" + }, + { + "test": "decompression bomb (400M px header, tiny body) -> 413 before decoding", + "result": "PASS", + "detail": "83 B rejected in 25 ms" + }, + { + "test": "corrupted images and MIME spoofing -> 415, never 500", + "result": "PASS", + "detail": "9 spoof/corrupt variants -> 415" + }, + { + "test": "dangerous filenames -> 400; Unicode display name kept as-is", + "result": "PASS", + "detail": "12 bad names rejected" + }, + { + "test": "attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400", + "result": "PASS", + "detail": null + }, + { + "test": "idempotency: missing attachment_ids == []; other attachment list -> 409", + "result": "PASS", + "detail": null + }, + { + "test": "10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back", + "result": "PASS", + "detail": "10 uploads via 2 replicas" + }, + { + "test": "ACL: outsider gets 404 on upload, metadata, content; no token -> 401", + "result": "PASS", + "detail": null + }, + { + "test": "message attachment_ids: other chat / foreign / unknown -> 404, nothing stored", + "result": "PASS", + "detail": null + } +] diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..167fc46d --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,19 @@ +PASS one session served by >=2 replicas without re-login (no stickiness) - distribution [15, 15] +PASS X-Instance-Id is present on errors and health too - +PASS Idempotency-Key missing or malformed -> 400, nothing stored - +PASS concurrent replay of one key through different replicas -> exactly one message - 24 parallel requests via 2 replicas -> 1 message +PASS replay is semantic: JSON key order/whitespace does not matter; other body -> 409 - +PASS key scope = user + chat: same key is independent for other user / other chat - +PASS logout is visible on every replica immediately (shared sessions) - revoked token rejected by 2 replicas +PASS replay re-checks the session and chat access - +PASS PNG and JPEG upload/download byte-exact with true Content-Type - png 111817 B, jpeg 2565 B +PASS size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413 - accepted 10420335 B, rejected 10485761 B +PASS decompression bomb (400M px header, tiny body) -> 413 before decoding - 83 B rejected in 25 ms +PASS corrupted images and MIME spoofing -> 415, never 500 - 9 spoof/corrupt variants -> 415 +PASS dangerous filenames -> 400; Unicode display name kept as-is - 12 bad names rejected +PASS attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400 - +PASS idempotency: missing attachment_ids == []; other attachment list -> 409 - +PASS 10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back - 10 uploads via 2 replicas +PASS ACL: outsider gets 404 on upload, metadata, content; no token -> 401 - +PASS message attachment_ids: other chat / foreign / unknown -> 404, nothing stored - +18/18 passed diff --git a/evidence/track2/cost.json b/evidence/track2/cost.json new file mode 100644 index 00000000..de509ef5 --- /dev/null +++ b/evidence/track2/cost.json @@ -0,0 +1,120 @@ +{ + "command": "cost", + "date_utc": "2026-10-29T10:33:43Z", + "commit": "ec540a1", + "result": { + "object_size_bytes": 100, + "reconciler_limits": "0.5 CPU, 128M", + "s3": "SeaweedFS 3.80, 1 CPU, 1G", + "runs": [ + { + "bench_objects": 1000, + "repeat": 1, + "objects_scanned": 1229, + "rows_scanned": 1232, + "list_ms": 52, + "db_ms": 9, + "total_ms": 61, + "per_object_us": 50.0, + "container_memory_peak_mib": 10.7, + "findings": 0 + }, + { + "bench_objects": 1000, + "repeat": 2, + "objects_scanned": 1229, + "rows_scanned": 1232, + "list_ms": 52, + "db_ms": 6, + "total_ms": 59, + "per_object_us": 48.1, + "container_memory_peak_mib": 10.1, + "findings": 0 + }, + { + "bench_objects": 1000, + "repeat": 3, + "objects_scanned": 1229, + "rows_scanned": 1232, + "list_ms": 52, + "db_ms": 7, + "total_ms": 60, + "per_object_us": 48.8, + "container_memory_peak_mib": 10.9, + "findings": 0 + }, + { + "bench_objects": 5000, + "repeat": 1, + "objects_scanned": 5229, + "rows_scanned": 5232, + "list_ms": 258, + "db_ms": 67, + "total_ms": 326, + "per_object_us": 62.5, + "container_memory_peak_mib": 17.2, + "findings": 0 + }, + { + "bench_objects": 5000, + "repeat": 2, + "objects_scanned": 5229, + "rows_scanned": 5232, + "list_ms": 345, + "db_ms": 62, + "total_ms": 408, + "per_object_us": 78.2, + "container_memory_peak_mib": 16.9, + "findings": 0 + }, + { + "bench_objects": 5000, + "repeat": 3, + "objects_scanned": 5229, + "rows_scanned": 5232, + "list_ms": 263, + "db_ms": 70, + "total_ms": 335, + "per_object_us": 64.1, + "container_memory_peak_mib": 15.8, + "findings": 0 + }, + { + "bench_objects": 20000, + "repeat": 1, + "objects_scanned": 20229, + "rows_scanned": 20232, + "list_ms": 1425, + "db_ms": 95, + "total_ms": 1523, + "per_object_us": 75.3, + "container_memory_peak_mib": 30.8, + "findings": 0 + }, + { + "bench_objects": 20000, + "repeat": 2, + "objects_scanned": 20229, + "rows_scanned": 20232, + "list_ms": 1537, + "db_ms": 41, + "total_ms": 1618, + "per_object_us": 80.0, + "container_memory_peak_mib": 31.1, + "findings": 0 + }, + { + "bench_objects": 20000, + "repeat": 3, + "objects_scanned": 20229, + "rows_scanned": 20232, + "list_ms": 1321, + "db_ms": 76, + "total_ms": 1404, + "per_object_us": 69.4, + "container_memory_peak_mib": 31.2, + "findings": 0 + } + ] + } +} diff --git a/evidence/track2/faults.json b/evidence/track2/faults.json new file mode 100644 index 00000000..fd401442 --- /dev/null +++ b/evidence/track2/faults.json @@ -0,0 +1,244 @@ +{ + "command": "faults", + "date_utc": "2026-10-29T10:31:36Z", + "commit": "ec540a1", + "result": { + "grace_used": "1m", + "faults": { + "F1_orphan_object_no_row": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69", + "F2_metadata_row_lost(U3)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed", + "F3_object_lost_of_used(U2)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad", + "F4_crash_after_put_before_ready": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a", + "F5_crash_before_put": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5", + "F6_F7_fresh_upload_in_progress": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/693c91e3-43f6-46f6-9bef-2b0de987546a", + "F8_fresh_orphan_object": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc" + }, + "steps": [ + { + "step": "dry-run", + "report": { + "mode": "dry-run", + "objects_scanned": 233, + "rows_scanned": 235, + "orphan_objects": 3, + "fresh_skipped": 1, + "stale_pending": 3, + "missing_objects": 1, + "size_mismatch": 0, + "recovered": 0, + "fixed": 0, + "errors": 0, + "total_ms": 34 + }, + "items": [ + { + "kind": "stale_pending", + "key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/34bf2792-f4dd-4fc8-b242-2f2ed7b49209", + "attachment_id": "34bf2792-f4dd-4fc8-b242-2f2ed7b49209", + "action": "would delete row and object" + }, + { + "kind": "stale_pending", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a", + "attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a", + "action": "would delete row and object" + }, + { + "kind": "stale_pending", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5", + "attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5", + "action": "would delete row and object" + }, + { + "kind": "missing_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad", + "attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad", + "action": "would mark missing (metadata kept)" + }, + { + "kind": "orphan_object", + "key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/5585fa4c-c149-49f6-b5d1-1196038eba43", + "action": "would delete object" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc", + "action": "skip: younger than grace" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed", + "action": "would delete object" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69", + "action": "would delete object" + } + ], + "state_before": { + "s3_objects": 233, + "db_rows": 235, + "db_pending": 4, + "db_missing_marked": 4 + }, + "state_after": { + "s3_objects": 233, + "db_rows": 235, + "db_pending": 4, + "db_missing_marked": 4 + }, + "dry_run_changed_nothing": true, + "this_experiment": { + "stale_pending": 2, + "missing_object": 1, + "orphan_object/skip": 1, + "orphan_object": 2 + }, + "other_findings_outside_experiment": { + "stale_pending": 1, + "orphan_object": 1 + } + }, + { + "step": "apply #1", + "report": { + "mode": "apply", + "objects_scanned": 233, + "rows_scanned": 235, + "orphan_objects": 3, + "fresh_skipped": 1, + "stale_pending": 3, + "missing_objects": 1, + "size_mismatch": 0, + "recovered": 0, + "fixed": 7, + "errors": 0, + "total_ms": 46 + }, + "this_experiment": { + "stale_pending": 2, + "missing_object": 1, + "orphan_object/skip": 1, + "orphan_object": 2 + }, + "items": [ + { + "kind": "stale_pending", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a", + "attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a", + "action": "deleted row and object" + }, + { + "kind": "stale_pending", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5", + "attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5", + "action": "deleted row and object" + }, + { + "kind": "missing_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad", + "attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad", + "action": "marked missing (metadata kept)" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc", + "action": "skip: younger than grace" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed", + "action": "deleted object" + }, + { + "kind": "orphan_object", + "key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69", + "action": "deleted object" + } + ], + "state_after": { + "s3_objects": 228, + "db_rows": 232, + "db_pending": 1, + "db_missing_marked": 5 + } + }, + { + "step": "apply #2 (repeat)", + "report": { + "mode": "apply", + "objects_scanned": 228, + "rows_scanned": 232, + "orphan_objects": 0, + "fresh_skipped": 1, + "stale_pending": 0, + "missing_objects": 0, + "size_mismatch": 0, + "recovered": 0, + "fixed": 0, + "errors": 0, + "total_ms": 25 + }, + "this_experiment": { + "orphan_object/skip": 1 + }, + "state_after": { + "s3_objects": 228, + "db_rows": 232, + "db_pending": 1, + "db_missing_marked": 5 + } + }, + { + "step": "apply after restoring U2 object", + "report": { + "mode": "apply", + "objects_scanned": 229, + "rows_scanned": 232, + "orphan_objects": 0, + "fresh_skipped": 1, + "stale_pending": 0, + "missing_objects": 0, + "size_mismatch": 0, + "recovered": 1, + "fixed": 1, + "errors": 0, + "total_ms": 27 + } + }, + { + "step": "dry-run with production grace 15m", + "report": { + "mode": "dry-run", + "objects_scanned": 229, + "rows_scanned": 232, + "orphan_objects": 0, + "fresh_skipped": 1, + "stale_pending": 0, + "missing_objects": 0, + "size_mismatch": 0, + "recovered": 0, + "fixed": 0, + "errors": 0, + "total_ms": 31 + } + } + ], + "verify": { + "U1_used_healthy_download_sha_ok": true, + "U2_metadata_kept": 200, + "U2_content_status": 404, + "U2_marked_missing": true, + "message_still_references_U1_U2": true, + "F1_orphan_deleted": true, + "F2_orphan_of_lost_row_deleted": true, + "F4_row_and_object_deleted": true, + "F5_row_deleted": true, + "F7_fresh_pending_row_kept": true, + "F7_fresh_object_kept": true, + "F8_fresh_orphan_kept": true, + "U2_restored_and_unmarked": true + } + } +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..e8c33608 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,45 @@ +global + log stdout format raw local0 info + maxconn 4096 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 5s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s + timeout client 75s + timeout server 75s + timeout http-request 5s + timeout http-keep-alive 10s + timeout check 500ms + retries 2 + retry-on conn-failure + option redispatch 1 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4 + server-template api 8 api:8080 + +frontend internal_stats + bind :8404 + monitor-uri /healthz + stats enable + stats uri /stats + stats refresh 5s diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..6e50813e --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,8 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +save "" +appendonly no +maxmemory 64mb +maxmemory-policy volatile-ttl +include /run/secrets/valkey_auth diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..608a27a8 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import json +import secrets +import shutil +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / 'secrets' + + +def write(name, value): + path = SECRETS / name + if path.exists(): + print('keep secrets/' + name) + return path.read_text().strip() + path.write_bytes((value + '\n').encode('utf-8')) + print('created secrets/' + name) + return value + + +def main(): + SECRETS.mkdir(exist_ok=True) + pg = write('pg_password', secrets.token_urlsafe(24)) + write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg) + vk_conf = SECRETS / 'valkey_auth.conf' + if vk_conf.exists(): + vk = vk_conf.read_text().split()[-1] + print('keep secrets/valkey_auth.conf') + else: + vk = secrets.token_urlsafe(24) + write('valkey_auth.conf', 'requirepass ' + vk) + write('cache_url', 'redis://:%s@valkey:6379/0' % vk) + # Two S3 identities: admin (bucket bootstrap only) and api (read/write/list + # of the one bucket). SeaweedFS has no anonymous identity -> bucket is private. + admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8)) + admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30)) + api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8)) + api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30)) + bucket = 'messenger-attachments' + identities = {'identities': [ + {'name': 'bootstrap-admin', + 'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}], + 'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']}, + {'name': 'messenger-api', + 'credentials': [{'accessKey': api_id, 'secretKey': api_secret}], + 'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]}, + ]} + write('s3_identities.json', json.dumps(identities, indent=2)) + env = ROOT / '.env' + if not env.exists(): + shutil.copy(ROOT / '.env.example', env) + print('created .env from .env.example') + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4ops.py b/scripts/lab4ops.py new file mode 100644 index 00000000..e8a635df --- /dev/null +++ b/scripts/lab4ops.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Operator checks for lab4 that need Docker access (run on the host, explicitly). + + python scripts/lab4ops.py private # bucket privacy outside the API + python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other + python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working + python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original + python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent + python scripts/lab4ops.py memory # peak RSS under concurrent uploads + +Each command prints JSON and, with --output, stores it in evidence/. +Synthetic data only; tokens/credentials are never printed. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import threading +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +sys.path.insert(0, str(ROOT / 'student-tests')) +sys.path.insert(0, str(ROOT / 'scripts')) +from client import Client, Failure, check # noqa: E402 +import imagegen # noqa: E402 +from lablib import api_containers, sh, wait_healthy # noqa: E402 + +PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() +NETWORK = 'mtusi-lab4_data' +TOOL_IMAGE = 'python:3.12.8-alpine3.21' + + +def winpath(p): + s = str(p) + return s.replace('\\', '/') + + +def s3tool(*args, stdin=None): + """Run s3tool.py in a throwaway container on the data network (S3 is not published).""" + cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK, + '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', + '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', + TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args] + r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8') + if r.returncode != 0: + raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:])) + return json.loads(r.stdout.strip().splitlines()[-1]) + + +def psql(sql): + return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', + '-At', '-v', 'ON_ERROR_STOP=1', '-c', sql) + + +def upload(c, actor, chat, data=PNG, name='sample.png', expected=201): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected, + actor=actor, raw=data, headers={'Content-Type': 'image/png'}, + schema='Attachment' if expected == 201 else None) + + +def cmd_private(a): + c = Client(a.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat)['json'] + key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id']) + res = { + 'object_key_shape': 'attachments// (server generated)', + 'key_contains_filename': 'sample' in key, + 'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'], + 'anonymous_list': None, + 'api_identity_get': s3tool('get', key)['status'], + 'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'], + 'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'], + 'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'], + 'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None, + 'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'], + 'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'], + } + try: + s3tool('list', 'attachments/', '--as', 'anonymous') + res['anonymous_list'] = 200 + except RuntimeError as exc: + res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip() + # the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there + s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin') + res['api_identity_put_other_bucket'] = subprocess.run( + ['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK, + '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', + TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'], + capture_output=True, text=True).stdout.strip() + check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy') + check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'], + 'api identity has more rights than expected') + return res + + +def cmd_failover_upload(a): + c = Client(a.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + r = upload(c, alice, chat) + att, served_by = r['json'], r['headers']['X-Instance-Id'] + target = [x for x in api_containers() if x['id'].startswith(served_by[:12])] + check(len(target) == 1, 'cannot map X-Instance-Id to one api container') + before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}') + sh('docker', 'stop', '-t', '15', target[0]['id']) + t0 = time.monotonic() + reads = [] + try: + for _ in range(10): + try: + d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + reads.append({'instance': d['headers']['X-Instance-Id'], + 'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']}) + except Failure as exc: + reads.append({'error': str(exc)[-60:]}) + time.sleep(0.3) + during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}') + finally: + sh('docker', 'start', target[0]['id']) + wait_healthy(target[0]['id']) + ok = [x for x in reads if x.get('sha256_ok')] + check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica') + return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads, + 'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during} + + +def cmd_s3_outage(a): + c = Client(a.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat)['json'] + pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'") + sh('docker', 'compose', 'stop', 's3') + res = {} + try: + u = upload(c, alice, chat, expected=503) + res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000) + d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000) + res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status'] + res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status'] + res['ready_status'] = c.call('GET', '/health/ready')['status'] + # expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler + res['pending_rows_left'] = int(psql( + "SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before) + finally: + sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3') + t0 = time.monotonic() + while True: + try: + upload(c, alice, chat) + break + except Failure: + check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start') + time.sleep(0.5) + res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1) + d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256'] + check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200, + 'S3 outage is not isolated to attachment endpoints') + return res + + +def cmd_no_local_file(a): + out = {} + for r in api_containers(): + mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id'])) + ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id']) + probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true') + # Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf + files = sh('docker', 'exec', r['id'], 'sh', '-c', + 'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" ' + '! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf ' + '! -name .dockerenv 2>/dev/null | head -20') + out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe, + 'mounts': [m.get('Destination') for m in mounts], + 'files_written_since_start': files.splitlines()} + check(ro == 'true' and not files.strip(), 'API container can store files locally') + return out + + +def cmd_bootstrap(a): + runs = [] + for _ in range(2): + for svc in ('migrate', 's3-init'): + log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc) + runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])}) + return runs + + +def peak(cid): + v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split() + return int(v[0]) / 2**20, int(v[1]) / 2**20 + + +def cmd_memory(a): + big = imagegen.near_limit_png() + c = Client(a.base_url, timeout=120) + alice, bob, eve, chat = c.bootstrap() + results = [] + for conc in a.levels: + for rep in range(a.repeat): + sh('docker', 'compose', 'restart', 'api') + for r in api_containers(): + check(wait_healthy(r['id'], 60), 'api not healthy') + time.sleep(1) + s3_id = sh('docker', 'compose', 'ps', '-q', 's3') + s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) + upload(c, alice, chat, big) # warm-up allocations of one replica + base = {r['name']: peak(r['id']) for r in api_containers()} + barrier = threading.Barrier(conc) + + def one(i): + local = Client(a.base_url, timeout=120) + barrier.wait() + try: + x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png', + [201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'}) + return x['status'], x['elapsed'] + except Failure as exc: + return str(exc)[-40:], 0 + + t0 = time.monotonic() + with ThreadPoolExecutor(conc) as pool: + res = list(pool.map(one, range(conc))) + wall = time.monotonic() - t0 + after = {r['name']: peak(r['id']) for r in api_containers()} + lat = sorted(e for s, e in res if s == 201) + row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big), + 'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503), + 'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2), + 'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None, + 'max_ms': round(1000 * lat[-1]) if lat else None, + 'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()}, + 'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}} + row['max_peak_mib'] = max(row['peak_mib_per_replica'].values()) + row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1) + row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts + print(json.dumps(row, ensure_ascii=False), flush=True) + results.append(row) + return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB', + 'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128, + 's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')), + 'runs': results} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory']) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output') + p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40]) + p.add_argument('--repeat', type=int, default=2) + a = p.parse_args() + fn = globals()['cmd_' + a.cmd.replace('-', '_')] + result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)} + text = json.dumps(result, ensure_ascii=False, indent=2) + print(text) + if a.output: + Path(a.output).write_text(text + '\n', encoding='utf-8') + print('PASS: ' + a.cmd) + + +if __name__ == '__main__': + main() diff --git a/scripts/lablib.py b/scripts/lablib.py new file mode 100644 index 00000000..c22f41d2 --- /dev/null +++ b/scripts/lablib.py @@ -0,0 +1,130 @@ +import json +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure + +PROJECT = 'mtusi-lab4' + + +def sh(*args, check=True): + r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8') + if check and r.returncode != 0: + raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300])) + return r.stdout.strip() + + +def api_containers(all_states=False): + out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc', + '--filter', 'label=com.docker.compose.project=' + PROJECT, + '--filter', 'label=com.docker.compose.service=api', + '--format', '{{json .}}') + rows = [] + for line in out.splitlines(): + d = json.loads(line) + rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']}) + return sorted(rows, key=lambda r: r['name']) + + +def health(cid): + return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid) + + +def wait_healthy(cid, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if health(cid) == 'healthy': + return True + time.sleep(0.25) + return False + + +def ps_snapshot(): + return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}') + + +class WriteLoad: + + def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0): + self.base = base_url + self.workers = workers + self.timeout = timeout + self.pause = pause + self.events = [] + self.confirmed = [] + self.retries = 0 + self._lock = threading.Lock() + self._stop = threading.Event() + self._threads = [] + c = Client(base_url, timeout=10) + self.alice, self.bob, _eve, self.chat = c.bootstrap() + self.reader = c + + def _worker(self, wid): + c = Client(self.base, timeout=self.timeout) + key = text = None + n = 0 + while not self._stop.is_set(): + if key is None: + key = uuid.uuid4().hex + text = 'load-%d-%d-%s' % (wid, n, key[:8]) + start = time.monotonic() + try: + r = c.message(self.alice, self.chat, text, key=key) + with self._lock: + self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start)) + self.confirmed.append(r['json']) + key = None + n += 1 + except Failure as exc: + with self._lock: + self.events.append((start, False, str(exc)[-60:], time.monotonic() - start)) + self.retries += 1 + time.sleep(0.05) + if self.pause: + time.sleep(self.pause) + + def start(self): + for i in range(self.workers): + t = threading.Thread(target=self._worker, args=(i,), daemon=True) + t.start() + self._threads.append(t) + self.t0 = time.monotonic() + + def stop(self): + self._stop.set() + for t in self._threads: + t.join(timeout=15) + + def window(self, a, b): + ev = [e for e in self.events if a <= e[0] < b] + ok = [e for e in ev if e[1]] + lat = sorted(e[3] for e in ok) + pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None + return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok), + 'ok_rps': round(len(ok) / (b - a), 1) if b > a else None, + 'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)} + + def verify_history(self): + items, cursor = [], None + path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100' + while True: + page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''), + actor=self.bob, schema='MessagePage')['json'] + items += page['items'] + cursor = page['next_cursor'] + if not cursor: + break + by_id = {m['id']: m for m in items} + texts = [m['text'] for m in items] + lost = [m for m in self.confirmed if by_id.get(m['id']) != m] + return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed), + 'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id), + 'duplicate_texts': len(texts) - len(set(texts)), + 'unique_confirmed': len({m['id'] for m in self.confirmed})} diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..fc38c4fe --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +cd /d "%~dp0.." +docker compose stop lb api reconciler || exit /b 1 +docker compose restart postgres valkey s3 || exit /b 1 +docker compose up -d --no-recreate --wait postgres valkey s3 || exit /b 1 +docker compose start api reconciler lb || exit /b 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a360fcd0 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# Restart API and ALL state stores (PostgreSQL, Valkey, S3) keeping volumes. +set -eu +cd "$(dirname "$0")/.." +docker compose stop lb api reconciler +docker compose restart postgres valkey s3 +# no recreation, only waits until the stores report healthy again +docker compose up -d --no-recreate --wait postgres valkey s3 +docker compose start api reconciler lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..f2a75760 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,10 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2 +set CID= +for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api"') do set CID=%%i +if "%CID%"=="" exit /b 3 +docker stop -t 15 %CID% >nul || exit /b 1 +echo stopped api container %CID% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..004fb7db --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;; +esac +cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \ + --filter "label=com.docker.compose.project=mtusi-lab4" \ + --filter "label=com.docker.compose.service=api") +[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; } +[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; } +docker stop -t 15 "$cid" >/dev/null +echo "stopped api container $cid" diff --git a/scripts/track2/run_track2.py b/scripts/track2/run_track2.py new file mode 100644 index 00000000..f3b3a0a4 --- /dev/null +++ b/scripts/track2/run_track2.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost. + + python scripts/track2/run_track2.py faults --output evidence/track2/faults.json + python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json + +The periodic `reconciler` service is stopped for the duration (it would race the +experiment) and started again at the end. Synthetic data only. +""" +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'scripts')) +from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402 +from lablib import sh # noqa: E402 +from client import Client, check # noqa: E402 + + +def s3put_bytes(key, data): + cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK, + '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', + TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin'] + r = subprocess.run(cmd, input=data, capture_output=True) + check(r.returncode == 0, 's3 put failed') + return json.loads(r.stdout.decode().strip().splitlines()[-1]) + + +def reconcile(*args): + """One reconciler pass in a throwaway container; also returns its cgroup memory peak.""" + out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c', + '/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json ' + '&& echo && cat /sys/fs/cgroup/memory.peak') + lines = [x for x in out.splitlines() if x.strip()] + rep = json.loads(lines[-2]) + rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1) + return rep + + +def counts(prefix='attachments/'): + return {'s3_objects': s3tool('count', prefix)['count'], + 'db_rows': int(psql('SELECT count(*) FROM attachments')), + 'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")), + 'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))} + + +def summary(rep): + keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending', + 'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms') + return {k: rep[k] for k in keys} + + +def scoped(rep, prefix, invert=False): + out = {} + for i in rep['items']: + if i.get('key', '').startswith(prefix) == invert: + continue + k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '') + out[k] = out.get(k, 0) + 1 + return out + + +def insert_pending(att_id, chat, owner, key, age): + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height," + " object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100," + " repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')" + % (att_id, chat, owner, key, age, age)) + + +def exists(key): + return s3tool('get', key)['status'] == 200 + + +def cmd_faults(a): + c = Client(a.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/' + key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id) + + # healthy, referenced attachment (must survive everything) + u1 = upload(c, alice, chat)['json'] + # referenced attachment whose object will be lost (S3-side fault) + u2 = upload(c, alice, chat)['json'] + msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json'] + # unreferenced ready attachment whose metadata row will be lost (DB-side fault) + u3 = upload(c, alice, chat)['json'] + u3_key = key_of(u3['id']) + + f = {} + f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4()) + s3tool('put', f['F1_orphan_object_no_row']) + psql("DELETE FROM attachments WHERE id = '%s'" % u3['id']) + f['F2_metadata_row_lost(U3)'] = u3_key + s3tool('delete', key_of(u2['id'])) + f['F3_object_lost_of_used(U2)'] = key_of(u2['id']) + f4_id = str(uuid.uuid4()) + f['F4_crash_after_put_before_ready'] = prefix + f4_id + insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour') + s3tool('put', f['F4_crash_after_put_before_ready']) + f5_id = str(uuid.uuid4()) + f['F5_crash_before_put'] = prefix + f5_id + insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour') + + print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True) + time.sleep(a.wait) + # fresh state that must NOT be touched: an object being uploaded right now and its pending row + f7_id = str(uuid.uuid4()) + f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id + insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds') + s3tool('put', f['F6_F7_fresh_upload_in_progress']) + f8 = prefix + str(uuid.uuid4()) + f['F8_fresh_orphan_object'] = f8 + s3tool('put', f8) + + steps = [] + before = counts() + dry = reconcile('--grace', a.grace) + after_dry = counts() + steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before, + 'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry}) + check(before == after_dry, 'dry-run modified state') + mine = scoped(dry, prefix) + steps[-1]['this_experiment'] = mine + steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True) + check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1}, + 'detection mismatch: %s' % mine) + + ap = reconcile('--apply', '--grace', a.grace) + steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix), + 'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()}) + again = reconcile('--apply', '--grace', a.grace) + steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix), + 'state_after': counts()}) + check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op') + + d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob) + msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items'] + verify = { + 'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'], + 'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'], + 'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'], + 'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't', + 'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']], + 'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']), + 'F2_orphan_of_lost_row_deleted': not exists(u3_key), + 'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready']) + and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0', + 'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0', + 'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending', + 'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']), + 'F8_fresh_orphan_kept': exists(f8), + } + for k, v in verify.items(): + check(v is True or v in (200, 404), k) + + # restore U2 from a "backup" (same bytes): the reconciler clears the missing mark + s3put_bytes(key_of(u2['id']), PNG) + rec = reconcile('--apply', '--grace', a.grace) + d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob) + verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and + hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and + psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't') + steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)}) + check(verify['U2_restored_and_unmarked'], 'restore not detected') + + # default production grace (15m): the same kind of fresh garbage is only reported as skipped + prod = reconcile('--grace', '15m') + steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)}) + # clean the fresh leftovers with the regular procedure so the next run starts clean + return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify} + + +def cmd_cost(a): + c = Client(a.base_url, timeout=30) + alice, bob, eve, chat = c.bootstrap() + run = uuid.uuid4().hex[:8] + keys_file = ROOT / 'evidence/track2/.bench-keys.txt' + created = 0 + results = [] + try: + for n in sorted(a.sizes): + add = n - created + keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)] + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256," + " width, height, object_key, status, created_at, updated_at)" + " SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1," + " 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()" + " FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1)) + t0 = time.monotonic() + put = json.loads(subprocess.run( + ['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', + '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', + '/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'], + input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1]) + check(put['ok'] == add, 'bulk put incomplete: %s' % put) + created = n + print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True) + for r in range(a.repeat): + rep = reconcile('--grace', '15m') + row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'], + 'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'], + 'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1), + 'container_memory_peak_mib': rep['container_memory_peak_mib'], + 'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']} + print(json.dumps(row), flush=True) + results.append(row) + finally: + if created: + keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)] + psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run) + subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK, + '-v', winpath(ROOT / 'secrets') + ':/secrets:ro', + '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', + '/tool/s3tool.py', 'bulk-delete', '--workers', '32'], + input='\n'.join(keys), capture_output=True, text=True) + if keys_file.exists(): + keys_file.unlink() + return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G', + 'runs': results} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('cmd', choices=['faults', 'cost']) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--grace', default='1m') + p.add_argument('--wait', type=int, default=65) + p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000]) + p.add_argument('--repeat', type=int, default=3) + p.add_argument('--output') + a = p.parse_args() + sh('docker', 'compose', 'stop', 'reconciler') + try: + res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), + 'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)} + finally: + sh('docker', 'compose', 'start', 'reconciler') + text = json.dumps(res, ensure_ascii=False, indent=2) + if a.output: + Path(a.output).parent.mkdir(parents=True, exist_ok=True) + Path(a.output).write_text(text + '\n', encoding='utf-8') + print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1])) + print('PASS: track2 ' + a.cmd) + + +if __name__ == '__main__': + main() diff --git a/scripts/track2/s3tool.py b/scripts/track2/s3tool.py new file mode 100644 index 00000000..50269d50 --- /dev/null +++ b/scripts/track2/s3tool.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API. + +Runs inside a throwaway container on the compose `data` network (S3 is not +published to the host), see run_in_net() in lab4ops.py. Credentials are read +from mounted secret files and never printed. +""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import hmac +import http.client +import json +import os +import sys +import threading +from urllib.parse import quote +import xml.etree.ElementTree as ET + +HOST = os.environ.get('S3_HOST', 's3:8333') +REGION = os.environ.get('S3_REGION', 'us-east-1') +BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') +SECRETS = os.environ.get('SECRETS_DIR', '/secrets') +NS = '{http://s3.amazonaws.com/doc/2006-03-01/}' + + +def creds(identity): + if identity == 'anonymous': + return None + prefix = 's3_admin_' if identity == 'admin' else 's3_' + read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip() + return read('access_key_id'), read('secret_access_key') + + +def _h(key, msg): + return hmac.new(key, msg.encode(), hashlib.sha256).digest() + + +_local = threading.local() + + +def request(method, path, identity='api', body=b'', query=None): + query = query or {} + headers = {'host': HOST} + payload_hash = hashlib.sha256(body).hexdigest() + c = creds(identity) + if c: + now = dt.datetime.now(dt.timezone.utc) + amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d') + headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate}) + signed = ';'.join(sorted(headers)) + cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) + creq = '\n'.join([method, quote(path, safe='/-_.~'), cq, + ''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash]) + scope = '%s/%s/s3/aws4_request' % (date, REGION) + sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()]) + k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request') + sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest() + headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % ( + c[0], scope, signed, sig) + url = quote(path, safe='/-_.~') + if query: + url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items())) + conn = getattr(_local, 'conn', None) + if conn is None: + conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30) + try: + conn.request(method, url, body=body or None, headers=headers) + r = conn.getresponse() + return r.status, r.read() + except (OSError, http.client.HTTPException): + _local.conn = None + raise + + +def obj(key): + return '/' + BUCKET + '/' + key + + +def list_keys(prefix, identity='api'): + keys, token = [], None + while True: + q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'} + if token: + q['continuation-token'] = token + st, body = request('GET', '/' + BUCKET, identity, query=q) + if st != 200: + raise SystemExit('list failed: HTTP %d' % st) + root = ET.fromstring(body) + keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')] + if root.findtext(NS + 'IsTruncated') != 'true': + return keys + token = root.findtext(NS + 'NextContinuationToken') + + +def main(): + p = argparse.ArgumentParser() + p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete']) + p.add_argument('arg', nargs='?', default='') + p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous']) + p.add_argument('--size', type=int, default=100) + p.add_argument('--workers', type=int, default=32) + p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes') + a = p.parse_args() + out = {'cmd': a.cmd, 'as': a.identity} + if a.cmd == 'get': + st, body = request('GET', obj(a.arg), a.identity) + out.update(status=st, bytes=len(body) if st == 200 else 0, + sha256=hashlib.sha256(body).hexdigest() if st == 200 else None) + elif a.cmd == 'put': + body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size) + st, _ = request('PUT', obj(a.arg), a.identity, body=body) + out.update(status=st) + elif a.cmd == 'delete': + st, _ = request('DELETE', obj(a.arg), a.identity) + out.update(status=st) + elif a.cmd == 'create-bucket': + st, _ = request('PUT', '/' + a.arg, a.identity) + out.update(status=st) + elif a.cmd == 'list': + out.update(keys=list_keys(a.arg, a.identity)) + elif a.cmd == 'count': + out.update(count=len(list_keys(a.arg, a.identity))) + elif a.cmd in ('bulk-put', 'bulk-delete'): + keys = [k.strip() for k in sys.stdin if k.strip()] + data = b'x' * a.size + + def one(k): + if a.cmd == 'bulk-put': + return request('PUT', obj(k), a.identity, body=data)[0] + return request('DELETE', obj(k), a.identity)[0] + + start = dt.datetime.now() + with ThreadPoolExecutor(a.workers) as pool: + codes = list(pool.map(one, keys)) + out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)), + seconds=round((dt.datetime.now() - start).total_seconds(), 2)) + print(json.dumps(out)) + + +if __name__ == '__main__': + main() diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go new file mode 100644 index 00000000..57cc2bde --- /dev/null +++ b/src/cmd/server/main.go @@ -0,0 +1,331 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "log/slog" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "sync/atomic" + "syscall" + "time" + + "mtusi-messenger/internal/api" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/cache" + "mtusi-messenger/internal/db" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/reconcile" + "mtusi-messenger/internal/store" +) + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func getint(key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + fmt.Fprintf(os.Stderr, "invalid %s\n", key) + os.Exit(2) + } + return n +} + +func secret(key string) (string, error) { + if path := os.Getenv(key + "_FILE"); path != "" { + b, err := os.ReadFile(path) + if err != nil { + return "", fmt.Errorf("read %s_FILE: %w", key, err) + } + return strings.TrimSpace(string(b)), nil + } + if v := os.Getenv(key); v != "" { + return v, nil + } + return "", fmt.Errorf("%s or %s_FILE is required", key, key) +} + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) + if err := run(log); err != nil { + log.Error("fatal", "error", err.Error()) + os.Exit(1) + } +} + +func openDB(ctx context.Context) (*store.Store, func(), error) { + dsn, err := secret("DATABASE_URL") + if err != nil { + return nil, nil, err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))}) + if err != nil { + return nil, nil, errors.New("database config is invalid") + } + return store.New(pool), pool.Close, nil +} + +func getduration(key string, def time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil || d <= 0 { + fmt.Fprintf(os.Stderr, "invalid %s\n", key) + os.Exit(2) + } + return d +} + +func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") } + +// openObjectsAs reads _ACCESS_KEY_ID[_FILE] / _SECRET_ACCESS_KEY[_FILE]: +// S3 for the restricted API identity, S3_ADMIN only in the one-shot bootstrap. +func openObjectsAs(prefix string) (*objstore.Store, error) { + access, err := secret(prefix + "_ACCESS_KEY_ID") + if err != nil { + return nil, err + } + secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY") + if err != nil { + return nil, err + } + return objstore.New(objstore.Config{ + Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"), + Region: getenv("S3_REGION", "us-east-1"), + Bucket: getenv("S3_BUCKET", "messenger-attachments"), + AccessKey: access, + SecretKey: secretKey, + }) +} + +// runReconcile: `server reconcile [--apply] [--grace 15m] [--every 10m]`. +// Dry-run by default; prints one JSON report per pass. +func runReconcile(ctx context.Context, log *slog.Logger, args []string) error { + fs := flag.NewFlagSet("reconcile", flag.ContinueOnError) + apply := fs.Bool("apply", false, "apply fixes (default: dry-run, report only)") + grace := fs.Duration("grace", getduration("RECONCILE_GRACE", 15*time.Minute), "minimum age before an object/row is touched") + every := fs.Duration("every", 0, "repeat interval; 0 runs once") + if err := fs.Parse(args); err != nil { + return err + } + if *grace < time.Minute { + return errors.New("grace below 1m is unsafe: it is shorter than an upload may take") + } + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + objects, err := openObjects() + if err != nil { + return err + } + rec := reconcile.New(st, objects, log) + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + for { + pctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + rep, err := rec.Run(pctx, reconcile.Options{Apply: *apply, Grace: *grace}) + cancel() + if err != nil { + if *every == 0 { + return err + } + log.Warn("reconcile pass failed", "error", err.Error()) + } else { + out, _ := json.Marshal(rep) + fmt.Println(string(out)) + } + if *every == 0 { + return nil + } + select { + case <-stop: + return nil + case <-time.After(*every): + } + } +} + +func run(log *slog.Logger) error { + ctx := context.Background() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + dsn, err := secret("DATABASE_URL") + if err != nil { + return err + } + pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2}) + if err != nil { + return errors.New("database config is invalid") + } + defer pool.Close() + mctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + return db.Migrate(mctx, pool, log) + } + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + objects, err := openObjectsAs("S3_ADMIN") + if err != nil { + return err + } + var created bool + for attempt := 1; ; attempt++ { + bctx, cancel := context.WithTimeout(ctx, 5*time.Second) + created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1")) + cancel() + if err == nil || attempt == 30 { + break + } + log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error()) + time.Sleep(2 * time.Second) + } + if err != nil { + return err + } + log.Info("bucket ready", "bucket", objects.Bucket(), "created", created) + return nil + } + if len(os.Args) > 1 && os.Args[1] == "reconcile" { + return runReconcile(ctx, log, os.Args[2:]) + } + + st, closeDB, err := openDB(ctx) + if err != nil { + return err + } + defer closeDB() + + ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second + cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second + + var sessCache auth.SessionCache + switch mode := getenv("SESSION_STORE", "postgres"); mode { + case "postgres": + case "valkey", "redis": + url, err := secret("CACHE_URL") + if err != nil { + return err + } + vk, err := cache.NewValkey(url) + if err != nil { + return errors.New("cache config is invalid") + } + defer vk.Close() + sessCache = vk + default: + return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode) + } + + hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4)) + sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log) + + instanceID := getenv("INSTANCE_ID", "") + if instanceID == "" { + h, err := os.Hostname() + if err != nil || h == "" { + return errors.New("cannot determine instance id; set INSTANCE_ID") + } + instanceID = h + } + idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second + objects, err := openObjects() + if err != nil { + return err + } + maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760)) + if maxUpload > 10485760 { + return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit") + } + drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond + + ready := &atomic.Bool{} + ready.Store(true) + + host := getenv("APP_HOST", "0.0.0.0") + port := getenv("APP_PORT", "8080") + addr := host + ":" + port + srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{ + InstanceID: instanceID, IdempotencyTTL: idemTTL, + MaxUploadBytes: maxUpload, + MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)), + UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)), + DecodeBudgetBytes: int64(getint("DECODE_BUDGET_BYTES", 128<<20)), + UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second), + S3Timeout: getduration("S3_TIMEOUT", 15*time.Second), + }) + httpSrv := &http.Server{ + Addr: addr, + Handler: srv.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 90 * time.Second, + WriteTimeout: 90 * time.Second, + IdleTimeout: 60 * time.Second, + } + + bgCtx, stopBg := context.WithCancel(ctx) + defer stopBg() + go func() { + t := time.NewTicker(10 * time.Minute) + defer t.Stop() + for { + select { + case <-bgCtx.Done(): + return + case <-t.C: + c, cancel := context.WithTimeout(bgCtx, 10*time.Second) + if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 { + log.Info("expired sessions purged", "count", n) + } + if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 { + log.Info("expired idempotency keys purged", "count", n) + } + cancel() + } + } + }() + + errCh := make(chan error, 1) + go func() { errCh <- httpSrv.ListenAndServe() }() + log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(), + "session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds())) + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + + select { + case err := <-errCh: + if err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + case sig := <-stop: + log.Info("shutdown signal received", "signal", sig.String()) + ready.Store(false) + time.Sleep(drainDelay) + httpSrv.SetKeepAlivesEnabled(false) + sctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := httpSrv.Shutdown(sctx); err != nil { + log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error()) + _ = httpSrv.Close() + } + log.Info("server stopped cleanly") + } + return nil +} diff --git a/src/go.mod b/src/go.mod new file mode 100644 index 00000000..16edcacb --- /dev/null +++ b/src/go.mod @@ -0,0 +1,30 @@ +module mtusi-messenger + +go 1.23 + +require ( + github.com/google/uuid v1.6.0 + github.com/jackc/pgx/v5 v5.7.2 + github.com/minio/minio-go/v7 v7.0.82 + github.com/redis/go-redis/v9 v9.7.0 + golang.org/x/crypto v0.31.0 + golang.org/x/sync v0.10.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/rs/xid v1.6.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/text v0.21.0 // indirect +) diff --git a/src/go.sum b/src/go.sum new file mode 100644 index 00000000..7adf49f5 --- /dev/null +++ b/src/go.sum @@ -0,0 +1,62 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI= +github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo= +github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/internal/api/attachments.go b/src/internal/api/attachments.go new file mode 100644 index 00000000..50ab0845 --- /dev/null +++ b/src/internal/api/attachments.go @@ -0,0 +1,338 @@ +package api + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/color" + "image/jpeg" + "image/png" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/store" +) + +const ( + mimePNG = "image/png" + mimeJPEG = "image/jpeg" + // An oversized body is read and discarded up to this many bytes so the client + // receives 413 instead of a connection reset; beyond it the connection is closed. + maxDrainBytes = 64 << 20 +) + +func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) { + s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error()) + apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable")) +} + +func validFilename(name string) bool { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." { + return false + } + if n := runeLen(name); n < 1 || n > 255 { + return false + } + for _, c := range name { + switch { + case c == '/', c == '\\', c == 0, unicode.IsControl(c): + return false + case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069: // bidi overrides hide the real extension + return false + } + } + return true +} + +func sniffImage(data []byte) string { + switch { + case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")): + return mimePNG + case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}): + return mimeJPEG + } + return "" +} + +// decodeCost estimates bytes allocated by a full decode for the memory budget. +func decodeCost(m color.Model, pixels int64) int64 { + bpp := int64(4) + switch m { + case color.GrayModel, color.AlphaModel: + bpp = 1 + case color.Gray16Model, color.Alpha16Model: + bpp = 2 + case color.YCbCrModel: + bpp = 3 + case color.RGBA64Model, color.NRGBA64Model: + bpp = 8 + } + if _, ok := m.(color.Palette); ok { + bpp = 1 + } + return pixels * bpp +} + +type imageInfo struct { + contentType string + width, height int +} + +// inspectImage checks the real format by magic bytes, compares it with the +// declared Content-Type, enforces the pixel limit from the header only and then +// fully decodes the image under a shared memory budget to reject corrupt data. +func (s *Server) inspectImage(ctx context.Context, data []byte, declared string) (imageInfo, *apperr.AppError) { + detected := sniffImage(data) + if detected == "" { + return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image") + } + if detected != declared { + return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content") + } + cfg, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width <= 0 || cfg.Height <= 0 { + return imageInfo{}, apperr.UnsupportedMedia("image header is invalid") + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if pixels > s.maxPixels { + return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels") + } + cost := min(decodeCost(cfg.ColorModel, pixels), s.decodeBudget) + if err := s.decodeSem.Acquire(ctx, cost); err != nil { + return imageInfo{}, apperr.Unavailable("server is busy processing images, retry later") + } + if detected == mimePNG { + _, err = png.Decode(bytes.NewReader(data)) + } else { + _, err = jpeg.Decode(bytes.NewReader(data)) + } + s.decodeSem.Release(cost) + if err != nil { + return imageInfo{}, apperr.UnsupportedMedia("image data is corrupted") + } + return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil +} + +func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) { + var buf bytes.Buffer + if r.ContentLength > 0 { + buf.Grow(int(r.ContentLength)) + } + _, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit)) + var maxErr *http.MaxBytesError + switch { + case errors.As(err, &maxErr): + return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes") + case err != nil: + return nil, apperr.InvalidRequest("request body could not be read") + case buf.Len() == 0: + return nil, apperr.InvalidRequest("request body is empty") + } + return buf.Bytes(), nil +} + +func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + body := r.Body + defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }() + + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + query, err := url.ParseQuery(r.URL.RawQuery) + if err != nil || len(query["filename"]) != 1 { + apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required")) + return + } + filename := query.Get("filename") + if !validFilename(filename) { + apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters")) + return + } + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if media != mimePNG && media != mimeJPEG { + apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg")) + return + } + if r.ContentLength > s.maxUpload { + apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes")) + return + } + + // Each slot may hold up to maxUpload bytes in memory; waiting is bounded by the request timeout. + if err := s.uploadSem.Acquire(r.Context(), 1); err != nil { + apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later")) + return + } + defer s.uploadSem.Release(1) + + data, aerr := readBody(w, r, s.maxUpload) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + info, aerr := s.inspectImage(r.Context(), data, media) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + sum := sha256.Sum256(data) + attID := uuid.NewString() + att := &model.Attachment{ + ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename, + ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]), + Status: "ready", Variants: []model.Variant{}, + // The key is server-generated; filename never reaches S3. + ObjectKey: "attachments/" + chatID + "/" + attID, + } + + // 1. intent row (pending) -> 2. S3 PUT -> 3. pending->ready. A crash between + // the steps leaves a pending row and maybe an object; both are removed by the + // reconciler after the grace period, never an untracked object. + if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height); err != nil { + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + s.storeError(w, r, err) + return + } + putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout) + err = s.objects.Put(putCtx, att.ObjectKey, data, att.ContentType) + cancel() + if err != nil { + go s.compensateUpload(att, id) + s.objectError(w, r, err) + return + } + updated, err := s.store.MarkAttachmentReady(r.Context(), att.ID) + if err != nil { + go s.compensateUpload(att, id) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.Unavailable("upload expired, retry")) + return + } + s.storeError(w, r, err) + return + } + att.UpdatedAt = model.NewTime(updated) + writeJSON(w, r, http.StatusCreated, att) +} + +// compensateUpload is best effort and starts from the row: the object is removed +// only if this call deleted the still-pending row. If the row is already ready +// (the UPDATE committed but its reply was lost) the object is kept. Anything left +// behind (pending row while PostgreSQL is down, or an object whose removal +// failed) is cleaned up by the reconciler after the grace period. +func (s *Server) compensateUpload(att *model.Attachment, reqID string) { + dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second) + defer dcancel() + deleted, err := s.store.DeletePendingAttachment(dctx, att.ID) + if err != nil || !deleted { + if err != nil { + s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID, + "attachment_id", att.ID, "stage", "delete_row") + } + return + } + octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second) + defer ocancel() + if err := s.objects.Remove(octx, att.ObjectKey); err != nil { + s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID, + "attachment_id", att.ID, "stage", "remove_object") + } +} + +func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) { + id := requestID(r) + attID, err := parseUUID(r.PathValue("attachment_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID")) + return nil, false + } + att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible")) + return nil, false + } + if err != nil { + s.storeError(w, r, err) + return nil, false + } + return att, true +} + +func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + if att, ok := s.attachmentForRequest(w, r); ok { + writeJSON(w, r, http.StatusOK, att) + } +} + +func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + id := requestID(r) + att, ok := s.attachmentForRequest(w, r) + if !ok { + return + } + switch v := r.URL.Query().Get("variant"); v { + case "", "original": + case "thumbnail", "crop", "optimized": + apperr.Write(w, id, apperr.NotFound("variant not found")) + return + default: + apperr.Write(w, id, apperr.InvalidRequest("unknown variant")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout) + defer cancel() + obj, size, err := s.objects.Get(ctx, att.ObjectKey) + if objstore.IsNotFound(err) { + s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID) + apperr.Write(w, id, apperr.NotFound("attachment content is not available")) + return + } + if err != nil { + s.objectError(w, r, err) + return + } + defer obj.Close() + h := w.Header() + h.Set("Content-Type", att.ContentType) + h.Set("Content-Length", strconv.FormatInt(size, 10)) + h.Set("X-Content-Type-Options", "nosniff") + h.Set("Cache-Control", "private, no-store") + h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename)) + h.Set("ETag", `"`+att.SHA256+`"`) + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, obj); err != nil { + s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error()) + } +} diff --git a/src/internal/api/handlers.go b/src/internal/api/handlers.go new file mode 100644 index 00000000..8efd0191 --- /dev/null +++ b/src/internal/api/handlers.go @@ -0,0 +1,427 @@ +package api + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "strings" + "time" + "unicode/utf8" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) { + id := requestID(r) + if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) { + s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable")) + return + } + s.log.Error("unexpected storage error", "request_id", id, "error", err.Error()) + apperr.Write(w, id, apperr.Internal("internal error")) +} + +func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) { + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) { + if !s.ready.Load() { + apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down")) + return + } + ctx, cancel := context.WithTimeout(r.Context(), time.Second) + defer cancel() + if err := s.store.Ping(ctx); err != nil { + apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable")) + return + } + writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"}) +} + +type createUserReq struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return utf8.ValidString(p) && n >= 12 && n <= 128 +} + +func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + id := requestID(r) + var req createUserReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !usernameRe.MatchString(req.Username) { + apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$")) + return + } + if !validText(req.DisplayName, 100) { + apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points")) + return + } + if !validPassword(req.Password) { + apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points")) + return + } + hash, err := s.hasher.Hash(r.Context(), req.Password) + if err != nil { + s.storeError(w, r, err) + return + } + u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash) + if errors.Is(err, store.ErrUsernameTaken) { + apperr.Write(w, id, apperr.Conflict("username is already taken")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, u) +} + +func (s *Server) me(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + writeJSON(w, r, http.StatusOK, currentUser(r)) +} + +const basicChallenge = `Basic realm="messenger", charset="UTF-8"` + +func unauthorizedBasic(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", basicChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password")) +} + +func parseBasic(r *http.Request) (string, string, bool) { + scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Basic") { + return "", "", false + } + raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value)) + if err != nil || !utf8.Valid(raw) { + return "", "", false + } + user, pass, ok := strings.Cut(string(raw), ":") + if !ok || !usernameRe.MatchString(user) || !validPassword(pass) { + return "", "", false + } + return user, pass, true +} + +func (s *Server) createSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + methodNotAllowed(w, r) + return + } + username, password, ok := parseBasic(r) + if !ok { + unauthorizedBasic(w, r) + return + } + user, hash, err := s.store.Credentials(r.Context(), username) + if err != nil && !errors.Is(err, store.ErrNotFound) { + s.storeError(w, r, err) + return + } + + valid, err := s.hasher.Verify(r.Context(), password, hash) + if err != nil { + s.storeError(w, r, err) + return + } + if !valid { + unauthorizedBasic(w, r) + return + } + token, expires, err := s.sessions.Create(r.Context(), user) + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, model.Session{ + Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user, + }) +} + +func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodDelete { + methodNotAllowed(w, r) + return + } + err := s.sessions.Revoke(r.Context(), currentToken(r)) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(http.StatusNoContent) +} + +type createChatReq struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createChat(w, r) + case http.MethodGet: + s.listChats(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) createChat(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + var req createChatReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + apperr.Write(w, id, aerr) + return + } + if !validText(req.Title, 100) { + apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points")) + return + } + if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items")) + return + } + seen := make(map[string]bool, len(req.MemberIDs)) + for _, raw := range req.MemberIDs { + uid, err := parseUUID(raw) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings")) + return + } + if seen[uid] { + apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id")) + return + } + seen[uid] = true + } + caller := currentUser(r) + if !seen[caller.ID] { + apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids")) + return + } + + chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs) + if errors.Is(err, store.ErrUnknownMember) { + apperr.Write(w, id, apperr.InvalidRequest("unknown member_id")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusCreated, chat) +} + +func (s *Server) listChats(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + caller := currentUser(r) + limit, cur, aerr := pagingParams(r, caller.ID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Chat]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, r) + return + } + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return + } + chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID) + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + writeJSON(w, r, http.StatusOK, chat) +} + +type createMessageReq struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` +} + +func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPost: + s.createMessage(w, r) + case http.MethodGet: + s.listMessages(w, r) + default: + methodNotAllowed(w, r) + } +} + +func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) { + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return "", false + } + ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID) + if err != nil { + s.storeError(w, r, err) + return "", false + } + if !ok { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return "", false + } + return chatID, true +} + +func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, err := parseUUID(r.PathValue("chat_id")) + if err != nil { + apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID")) + return + } + // Validation errors are reported only to members; outsiders always get 404. + reject := func(aerr *apperr.AppError) { + if _, ok := s.chatAccess(w, r); !ok { + return + } + apperr.Write(w, id, aerr) + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}")) + return + } + var req createMessageReq + if aerr := decodeJSON(w, r, &req); aerr != nil { + reject(aerr) + return + } + if req.AttachmentIDs == nil { + req.AttachmentIDs = []string{} + } + if len(req.AttachmentIDs) > 10 { + reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items")) + return + } + seen := make(map[string]bool, len(req.AttachmentIDs)) + for _, raw := range req.AttachmentIDs { + aid, err := parseUUID(raw) + if err != nil { + reject(apperr.InvalidRequest("attachment_ids must be UUID strings")) + return + } + if seen[aid] { + reject(apperr.InvalidRequest("duplicate attachment_id")) + return + } + seen[aid] = true + } + if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) { + reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments")) + return + } + if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 { + reject(apperr.InvalidRequest("text must be at most 4000 code points")) + return + } + msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs, + key, messageFingerprint(req), s.idempotencyTTL) + if errors.Is(err, store.ErrIdempotencyConflict) { + apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request")) + return + } + if errors.Is(err, store.ErrNotFound) { + apperr.Write(w, id, apperr.NotFound("chat not found or not accessible")) + return + } + if errors.Is(err, store.ErrAttachmentNotFound) { + apperr.Write(w, id, apperr.NotFound("attachment not found in this chat")) + return + } + if err != nil { + s.storeError(w, r, err) + return + } + if replayed { + w.Header().Set("Idempotent-Replayed", "true") + } + writeJSON(w, r, http.StatusCreated, msg) +} + +func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) { + id := requestID(r) + chatID, ok := s.chatAccess(w, r) + if !ok { + return + } + limit, cur, aerr := pagingParams(r, chatID) + if aerr != nil { + apperr.Write(w, id, aerr) + return + } + items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur) + if err != nil { + s.storeError(w, r, err) + return + } + page := model.Page[*model.Message]{Items: items} + if more && len(items) > 0 { + last := items[len(items)-1] + encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID}) + page.NextCursor = &encoded + } + writeJSON(w, r, http.StatusOK, page) +} + +func (s *Server) notFound(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.NotFound("route not found")) +} diff --git a/src/internal/api/helpers.go b/src/internal/api/helpers.go new file mode 100644 index 00000000..4ed2c6da --- /dev/null +++ b/src/internal/api/helpers.go @@ -0,0 +1,110 @@ +package api + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/store" +) + +const maxBodyBytes = 1 << 20 + +var ( + usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) +) + +func messageFingerprint(req createMessageReq) []byte { + b, _ := json.Marshal(req) + sum := sha256.Sum256(b) + return sum[:] +} + +func parseUUID(s string) (string, error) { + u, err := uuid.Parse(s) + if err != nil || u.String() != s { + return "", errors.New("invalid uuid") + } + return u.String(), nil +} + +func runeLen(s string) int { return utf8.RuneCountInString(s) } + +func validText(s string, max int) bool { + n := runeLen(s) + return n >= 1 && n <= max && strings.TrimSpace(s) != "" +} + +func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError { + media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type")) + if media != "application/json" { + return apperr.UnsupportedMedia("Content-Type must be application/json") + } + r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return apperr.PayloadTooLarge("request body too large") + } + return apperr.InvalidRequest("malformed JSON body") + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + return apperr.PayloadTooLarge("request body too large") + } + return apperr.InvalidRequest("malformed JSON body") + } + return nil +} + +func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) { + limit := 50 + if v := r.URL.Query().Get("limit"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 1 || n > 100 { + return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100") + } + limit = n + } + var cur *store.Cursor + if v := r.URL.Query().Get("cursor"); v != "" { + c, err := store.DecodeCursor(v) + if err == nil { + _, err = parseUUID(c.ID) + } + if err != nil { + return 0, nil, apperr.InvalidRequest("invalid cursor") + } + if c.Scope != scope { + return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource") + } + cur = &c + } + return limit, cur, nil +} + +func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID(r)) + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(payload) +} + +func methodNotAllowed(w http.ResponseWriter, r *http.Request) { + apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed")) +} diff --git a/src/internal/api/middleware.go b/src/internal/api/middleware.go new file mode 100644 index 00000000..d9e69cb1 --- /dev/null +++ b/src/internal/api/middleware.go @@ -0,0 +1,150 @@ +package api + +import ( + "context" + "errors" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "mtusi-messenger/internal/apperr" + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/model" +) + +type ctxKey int + +const ( + ctxRequestID ctxKey = iota + ctxUser + ctxToken +) + +func requestIDMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + id := r.Header.Get("X-Request-Id") + if id == "" { + id = uuid.NewString() + } + w.Header().Set("X-Request-Id", id) + next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id))) + }) +} + +func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", instanceID) + next.ServeHTTP(w, r) + }) +} + +func requestID(r *http.Request) string { + if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" { + return id + } + return "unknown" +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (s *statusWriter) WriteHeader(code int) { + s.status = code + s.ResponseWriter.WriteHeader(code) +} + +func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + sw := &statusWriter{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(sw, r) + + log.Info("http", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "dur_ms", time.Since(start).Milliseconds(), + "request_id", requestID(r), + ) + }) +} + +func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec != nil { + log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r)) + apperr.Write(w, requestID(r), apperr.Internal("internal error")) + } + }() + next.ServeHTTP(w, r) + }) +} + +// Binary transfers get a longer budget than JSON requests. +func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + limit := d + if strings.Contains(r.URL.Path, "/attachments") { + limit = binary + } + ctx, cancel := context.WithTimeout(r.Context(), limit) + defer cancel() + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +const bearerChallenge = `Bearer realm="messenger"` + +func unauthorizedBearer(w http.ResponseWriter, r *http.Request) { + w.Header().Set("WWW-Authenticate", bearerChallenge) + apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required")) +} + +func bearerToken(r *http.Request) (string, bool) { + scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ") + if !ok || !strings.EqualFold(scheme, "Bearer") { + return "", false + } + token = strings.TrimSpace(token) + if token == "" || len(token) > 256 { + return "", false + } + return token, true +} + +func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, ok := bearerToken(r) + if !ok { + unauthorizedBearer(w, r) + return + } + user, err := s.sessions.Resolve(r.Context(), token) + if errors.Is(err, auth.ErrInvalidSession) { + unauthorizedBearer(w, r) + return + } + if err != nil { + + s.storeError(w, r, err) + return + } + ctx := context.WithValue(r.Context(), ctxUser, user) + ctx = context.WithValue(ctx, ctxToken, token) + next.ServeHTTP(w, r.WithContext(ctx)) + } +} + +func currentUser(r *http.Request) *model.User { + return r.Context().Value(ctxUser).(*model.User) +} + +func currentToken(r *http.Request) string { + return r.Context().Value(ctxToken).(string) +} diff --git a/src/internal/api/router.go b/src/internal/api/router.go new file mode 100644 index 00000000..aaeffe89 --- /dev/null +++ b/src/internal/api/router.go @@ -0,0 +1,80 @@ +package api + +import ( + "log/slog" + "net/http" + "sync/atomic" + "time" + + "golang.org/x/sync/semaphore" + + "mtusi-messenger/internal/auth" + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/store" +) + +type Server struct { + store *store.Store + objects *objstore.Store + sessions *auth.Sessions + hasher *auth.Hasher + log *slog.Logger + ready *atomic.Bool + timeout time.Duration + + instanceID string + idempotencyTTL time.Duration + + uploadTimeout time.Duration + s3Timeout time.Duration + maxUpload int64 + maxPixels int64 + decodeBudget int64 + uploadSem *semaphore.Weighted + decodeSem *semaphore.Weighted +} + +type Options struct { + InstanceID string + IdempotencyTTL time.Duration + + MaxUploadBytes int64 + MaxImagePixels int64 + UploadConcurrency int64 + DecodeBudgetBytes int64 + UploadTimeout time.Duration + S3Timeout time.Duration +} + +func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server { + return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second, + instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL, + uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout, + maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels, decodeBudget: opt.DecodeBudgetBytes, + uploadSem: semaphore.NewWeighted(opt.UploadConcurrency), decodeSem: semaphore.NewWeighted(opt.DecodeBudgetBytes)} +} + +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health/live", s.healthLive) + mux.HandleFunc("/health/ready", s.healthReady) + mux.HandleFunc("/api/v1/users", s.usersRoute) + mux.HandleFunc("/api/v1/auth/sessions", s.createSession) + mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession)) + mux.HandleFunc("/api/v1/users/me", s.auth(s.me)) + mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute)) + mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment)) + mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment)) + mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment)) + mux.HandleFunc("/", s.notFound) + + var h http.Handler = mux + h = timeoutMiddleware(s.timeout, s.uploadTimeout, h) + h = recoverMiddleware(s.log, h) + h = loggingMiddleware(s.log, h) + h = requestIDMiddleware(h) + h = instanceIDMiddleware(s.instanceID, h) + return h +} diff --git a/src/internal/apperr/errors.go b/src/internal/apperr/errors.go new file mode 100644 index 00000000..0c6d7c5e --- /dev/null +++ b/src/internal/apperr/errors.go @@ -0,0 +1,64 @@ +package apperr + +import ( + "encoding/json" + "net/http" +) + +type Code string + +const ( + CodeInvalidRequest Code = "invalid_request" + CodeUnauthorized Code = "unauthorized" + CodeNotFound Code = "not_found" + CodeConflict Code = "conflict" + CodePayloadTooLarge Code = "payload_too_large" + CodeUnsupportedMedia Code = "unsupported_media_type" + CodeRateLimited Code = "rate_limited" + CodeUnavailable Code = "unavailable" + CodeInternal Code = "internal_error" +) + +type Body struct { + Code Code `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` +} + +type Envelope struct { + Error Body `json:"error"` +} + +type AppError struct { + Status int + Code Code + Msg string +} + +func (e *AppError) Error() string { return e.Msg } + +func New(status int, code Code, msg string) *AppError { + return &AppError{Status: status, Code: code, Msg: msg} +} + +func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) } +func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) } +func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) } +func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) } +func PayloadTooLarge(msg string) *AppError { + return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg) +} +func UnsupportedMedia(msg string) *AppError { + return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg) +} +func Unavailable(msg string) *AppError { + return New(http.StatusServiceUnavailable, CodeUnavailable, msg) +} +func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) } + +func Write(w http.ResponseWriter, requestID string, err *AppError) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("X-Request-Id", requestID) + w.WriteHeader(err.Status) + _ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}}) +} diff --git a/src/internal/auth/password.go b/src/internal/auth/password.go new file mode 100644 index 00000000..ceaf8fe4 --- /dev/null +++ b/src/internal/auth/password.go @@ -0,0 +1,100 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "errors" + "fmt" + "strings" + + "golang.org/x/crypto/argon2" +) + +const ( + argonMemoryKiB = 19 * 1024 + argonTime = 2 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +var ErrBadHash = errors.New("malformed password hash") + +type Hasher struct { + slots chan struct{} + dummy string +} + +func NewHasher(concurrency int) *Hasher { + h := &Hasher{slots: make(chan struct{}, concurrency)} + + salt := make([]byte, argonSaltLen) + _, _ = rand.Read(salt) + h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)) + return h +} + +func (h *Hasher) acquire(ctx context.Context) error { + select { + case h.slots <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (h *Hasher) release() { <-h.slots } + +func encode(salt, key []byte) string { + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads, + base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key)) +} + +func (h *Hasher) Hash(ctx context.Context, password string) (string, error) { + if err := h.acquire(ctx); err != nil { + return "", err + } + defer h.release() + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil +} + +func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) { + missing := encoded == "" + if missing { + encoded = h.dummy + } + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, ErrBadHash + } + var version int + var mem uint32 + var t uint32 + var p uint8 + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version { + return false, ErrBadHash + } + if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil { + return false, ErrBadHash + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, ErrBadHash + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, ErrBadHash + } + if err := h.acquire(ctx); err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want))) + h.release() + return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil +} diff --git a/src/internal/auth/sessions.go b/src/internal/auth/sessions.go new file mode 100644 index 00000000..cf0a51ec --- /dev/null +++ b/src/internal/auth/sessions.go @@ -0,0 +1,137 @@ +package auth + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "errors" + "log/slog" + "time" + + "mtusi-messenger/internal/model" + "mtusi-messenger/internal/store" +) + +var ( + ErrInvalidSession = errors.New("invalid session") + ErrCacheUnavailable = errors.New("session cache unavailable") +) + +type SessionCache interface { + Get(ctx context.Context, key string) (*CachedSession, error) + Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error + Del(ctx context.Context, key string) error +} + +type CachedSession struct { + User model.User `json:"user"` + ExpiresAt time.Time `json:"exp"` +} + +type Sessions struct { + store *store.Store + cache SessionCache + ttl time.Duration + cacheMaxTTL time.Duration + log *slog.Logger +} + +func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions { + return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log} +} + +func NewToken() (string, []byte, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, err + } + token := base64.RawURLEncoding.EncodeToString(raw) + return token, HashToken(token), nil +} + +func HashToken(token string) []byte { + sum := sha256.Sum256([]byte(token)) + return sum[:] +} + +func cacheKey(hash []byte) string { + return "sess:" + base64.RawURLEncoding.EncodeToString(hash) +} + +func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) { + token, hash, err := NewToken() + if err != nil { + return "", time.Time{}, err + } + expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl) + if err != nil { + return "", time.Time{}, err + } + s.cacheSet(ctx, hash, user, expires) + return token, expires, nil +} + +func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) { + hash := HashToken(token) + if s.cache != nil { + cs, err := s.cache.Get(ctx, cacheKey(hash)) + if err == nil && cs != nil { + if time.Now().Before(cs.ExpiresAt) { + u := cs.User + return &u, nil + } + + } else if err != nil { + s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error()) + } + } + user, expires, err := s.store.SessionUser(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return nil, ErrInvalidSession + } + if err != nil { + return nil, err + } + s.cacheSet(ctx, hash, user, expires) + return user, nil +} + +func (s *Sessions) Revoke(ctx context.Context, token string) error { + hash := HashToken(token) + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + return ErrCacheUnavailable + } + } + err := s.store.RevokeSession(ctx, hash) + if errors.Is(err, store.ErrNotFound) { + return ErrInvalidSession + } + if err != nil { + return err + } + if s.cache != nil { + if err := s.cache.Del(ctx, cacheKey(hash)); err != nil { + + s.log.Warn("session cache delete after revoke failed", "error", err.Error()) + } + } + return nil +} + +func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) { + if s.cache == nil { + return + } + ttl := time.Until(expires) + if ttl > s.cacheMaxTTL { + ttl = s.cacheMaxTTL + } + if ttl <= 0 { + return + } + if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil { + s.log.Debug("session cache set failed", "error", err.Error()) + } +} diff --git a/src/internal/cache/valkey.go b/src/internal/cache/valkey.go new file mode 100644 index 00000000..591d9e49 --- /dev/null +++ b/src/internal/cache/valkey.go @@ -0,0 +1,80 @@ +package cache + +import ( + "context" + "encoding/json" + "errors" + "sync/atomic" + "time" + + "github.com/redis/go-redis/v9" + + "mtusi-messenger/internal/auth" +) + +var ErrOpen = errors.New("cache circuit open") + +type Valkey struct { + rdb *redis.Client + openUntil atomic.Int64 + cooldown time.Duration +} + +func NewValkey(url string) (*Valkey, error) { + opt, err := redis.ParseURL(url) + if err != nil { + return nil, err + } + opt.DialTimeout = 500 * time.Millisecond + opt.ReadTimeout = 300 * time.Millisecond + opt.WriteTimeout = 300 * time.Millisecond + opt.PoolSize = 10 + opt.MaxRetries = 0 + return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil +} + +func (v *Valkey) Close() error { return v.rdb.Close() } + +func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) } + +func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() } + +func (v *Valkey) guard(err error) error { + if err != nil && !errors.Is(err, redis.Nil) { + v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano()) + } + return err +} + +func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) { + if v.open() { + return nil, ErrOpen + } + raw, err := v.rdb.Get(ctx, key).Bytes() + if errors.Is(err, redis.Nil) { + return nil, nil + } + if err := v.guard(err); err != nil { + return nil, err + } + var cs auth.CachedSession + if err := json.Unmarshal(raw, &cs); err != nil { + return nil, nil + } + return &cs, nil +} + +func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error { + if v.open() { + return ErrOpen + } + raw, err := json.Marshal(s) + if err != nil { + return err + } + return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err()) +} + +func (v *Valkey) Del(ctx context.Context, key string) error { + return v.guard(v.rdb.Del(ctx, key).Err()) +} diff --git a/src/internal/db/migrate.go b/src/internal/db/migrate.go new file mode 100644 index 00000000..96d0c779 --- /dev/null +++ b/src/internal/db/migrate.go @@ -0,0 +1,122 @@ +package db + +import ( + "context" + "embed" + "fmt" + "io/fs" + "log/slog" + "sort" + "strconv" + "strings" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +const advisoryLockID = 20260202 + +type migration struct { + version int + name string + sql string +} + +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, "migrations") + if err != nil { + return nil, err + } + var out []migration + for _, e := range entries { + name := e.Name() + prefix, _, ok := strings.Cut(name, "_") + if !ok || !strings.HasSuffix(name, ".sql") { + return nil, fmt.Errorf("bad migration file name %q", name) + } + v, err := strconv.Atoi(prefix) + if err != nil { + return nil, fmt.Errorf("bad migration version in %q", name) + } + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return nil, err + } + out = append(out, migration{version: v, name: name, sql: string(body)}) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + for i := 1; i < len(out); i++ { + if out[i].version == out[i-1].version { + return nil, fmt.Errorf("duplicate migration version %d", out[i].version) + } + } + return out, nil +} + +func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error { + migs, err := loadMigrations() + if err != nil { + return err + } + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + + if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil { + return err + } + defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID) + + if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations ( + version integer PRIMARY KEY, + name text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now())`); err != nil { + return err + } + + applied := map[int]bool{} + rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations") + if err != nil { + return err + } + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + rows.Close() + return err + } + applied[v] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + + for _, m := range migs { + if applied[m.version] { + continue + } + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, m.sql); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", m.name, err) + } + if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + log.Info("migration applied", "version", m.version, "name", m.name) + } + log.Info("migrations up to date", "total", len(migs)) + return nil +} diff --git a/src/internal/db/migrations/0001_init.sql b/src/internal/db/migrations/0001_init.sql new file mode 100644 index 00000000..a500b303 --- /dev/null +++ b/src/internal/db/migrations/0001_init.sql @@ -0,0 +1,51 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000), + CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE +); +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/src/internal/db/migrations/0002_message_idempotency.sql b/src/internal/db/migrations/0002_message_idempotency.sql new file mode 100644 index 00000000..5103b78d --- /dev/null +++ b/src/internal/db/migrations/0002_message_idempotency.sql @@ -0,0 +1,13 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32) +); +CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at); diff --git a/src/internal/db/migrations/0003_attachments.sql b/src/internal/db/migrations/0003_attachments.sql new file mode 100644 index 00000000..22c7c0ce --- /dev/null +++ b/src/internal/db/migrations/0003_attachments.sql @@ -0,0 +1,51 @@ +-- Lab4: attachment metadata lives here, bytes live in the private S3 bucket. +-- Text may be empty when a message carries at least one attachment (checked by the API). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); + +-- status: pending = intent row written before S3 PUT (not visible through the API); +-- ready = object stored and metadata committed. +-- missing_at is set by the reconciler when a ready row has no S3 object. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + object_key text NOT NULL, + status text NOT NULL, + missing_at timestamptz, + created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id), + CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')) +); +CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending'; + +-- (chat_id, attachment_id) -> attachments (chat_id, id): the database itself refuses +-- to link a file of another chat to a message. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id) + REFERENCES attachments (chat_id, id) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/src/internal/db/pool.go b/src/internal/db/pool.go new file mode 100644 index 00000000..3c818a19 --- /dev/null +++ b/src/internal/db/pool.go @@ -0,0 +1,31 @@ +package db + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type PoolOptions struct { + MaxConns int32 +} + +func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(url) + if err != nil { + return nil, err + } + cfg.MaxConns = opt.MaxConns + cfg.MinConns = 1 + cfg.MaxConnLifetime = 30 * time.Minute + cfg.MaxConnIdleTime = 5 * time.Minute + cfg.HealthCheckPeriod = 15 * time.Second + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + rp := cfg.ConnConfig.RuntimeParams + rp["application_name"] = "mtusi-messenger" + rp["statement_timeout"] = "5000" + rp["lock_timeout"] = "2000" + rp["idle_in_transaction_session_timeout"] = "10000" + return pgxpool.NewWithConfig(ctx, cfg) +} diff --git a/src/internal/model/models.go b/src/internal/model/models.go new file mode 100644 index 00000000..b3ece605 --- /dev/null +++ b/src/internal/model/models.go @@ -0,0 +1,79 @@ +package model + +import "time" + +type Time struct { + time.Time +} + +func NewTime(t time.Time) Time { return Time{Time: t} } + +func (t Time) MarshalJSON() ([]byte, error) { + return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil +} + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt Time `json:"created_at"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt Time `json:"created_at"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` +} + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt Time `json:"created_at"` + UpdatedAt Time `json:"updated_at"` + + ObjectKey string `json:"-"` + Missing bool `json:"-"` +} + +type Page[T any] struct { + Items []T `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type Health struct { + Status string `json:"status"` +} + +type Session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt Time `json:"expires_at"` + User User `json:"user"` +} diff --git a/src/internal/objstore/s3.go b/src/internal/objstore/s3.go new file mode 100644 index 00000000..01d438f5 --- /dev/null +++ b/src/internal/objstore/s3.go @@ -0,0 +1,151 @@ +package objstore + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Config struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string +} + +type Store struct { + c *minio.Client + bucket string +} + +type Object struct { + Key string + Size int64 + LastModified time.Time +} + +func New(cfg Config) (*Store, error) { + u, err := url.Parse(cfg.Endpoint) + if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") { + return nil, errors.New("S3_ENDPOINT must be http(s)://host:port") + } + if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" { + return nil, errors.New("S3 bucket and credentials are required") + } + // The SDK retries 10 times with backoff by default; a dead S3 must turn into a + // fast 503 instead (the client may retry the upload itself). + minio.MaxRetry = 1 + transport := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + MaxIdleConnsPerHost: 32, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + } + c, err := minio.New(u.Host, &minio.Options{ + Transport: transport, + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: u.Scheme == "https", + Region: cfg.Region, + BucketLookup: minio.BucketLookupPath, + }) + if err != nil { + return nil, err + } + return &Store{c: c, bucket: cfg.Bucket}, nil +} + +func (s *Store) Bucket() string { return s.bucket } + +// Ping checks that the bucket is reachable with the configured credentials. +func (s *Store) Ping(ctx context.Context) error { + ok, err := s.c.BucketExists(ctx, s.bucket) + if err != nil { + return err + } + if !ok { + return fmt.Errorf("bucket %q does not exist", s.bucket) + } + return nil +} + +// EnsureBucket creates the bucket if it is absent; safe to repeat. +func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) { + ok, err := s.c.BucketExists(ctx, s.bucket) + if err != nil || ok { + return false, err + } + if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil { + if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return false, nil + } + return false, err + } + return true, nil +} + +func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error { + _, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true}) + return err +} + +// Get returns a stream of the object; Stat is called first so that a missing +// object or an unavailable S3 is reported before any response byte is written. +func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) { + obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, 0, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, 0, err + } + return obj, info.Size, nil +} + +func (s *Store) Stat(ctx context.Context, key string) (Object, error) { + info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{}) + if err != nil { + return Object{}, err + } + return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil +} + +func (s *Store) Remove(ctx context.Context, key string) error { + return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error { + for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) { + if info.Err != nil { + return info.Err + } + if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil { + return err + } + } + return ctx.Err() +} + +func IsNotFound(err error) bool { + if err == nil { + return false + } + switch minio.ToErrorResponse(err).Code { + case "NoSuchKey", "NotFound": + return true + } + return false +} diff --git a/src/internal/reconcile/reconcile.go b/src/internal/reconcile/reconcile.go new file mode 100644 index 00000000..a9c668b3 --- /dev/null +++ b/src/internal/reconcile/reconcile.go @@ -0,0 +1,268 @@ +// Package reconcile compares attachment metadata in PostgreSQL with objects in +// the S3 bucket and repairs what can be repaired safely. +// +// Order matters: S3 is listed first, the database is read second. An upload +// writes its pending row before the PUT, so an object seen in the listing +// already has its row when the database is read. Every destructive decision is +// re-checked against the source of truth right before it is applied. +package reconcile + +import ( + "context" + "errors" + "log/slog" + "strings" + "time" + + "mtusi-messenger/internal/objstore" + "mtusi-messenger/internal/store" +) + +const ( + Prefix = "attachments/" + lockID = 20260404 +) + +type Options struct { + Apply bool + Grace time.Duration + Now func() time.Time +} + +type Item struct { + Kind string `json:"kind"` + Key string `json:"key,omitempty"` + ID string `json:"attachment_id,omitempty"` + Action string `json:"action"` +} + +type Report struct { + Mode string `json:"mode"` + GraceSeconds int64 `json:"grace_seconds"` + ObjectsScanned int `json:"objects_scanned"` + RowsScanned int `json:"rows_scanned"` + OrphanObjects int `json:"orphan_objects"` + FreshSkipped int `json:"fresh_skipped"` + StalePending int `json:"stale_pending"` + MissingObjects int `json:"missing_objects"` + SizeMismatch int `json:"size_mismatch"` + Recovered int `json:"recovered"` + Fixed int `json:"fixed"` + Errors int `json:"errors"` + ListMS int64 `json:"list_ms"` + DBMS int64 `json:"db_ms"` + TotalMS int64 `json:"total_ms"` + Items []Item `json:"items"` + Truncated bool `json:"items_truncated"` + PerObjectUS float64 `json:"per_object_us"` +} + +var ErrLocked = errors.New("another reconciler run holds the lock") + +type Reconciler struct { + db *store.Store + s3 *objstore.Store + log *slog.Logger +} + +func New(db *store.Store, s3 *objstore.Store, log *slog.Logger) *Reconciler { + return &Reconciler{db: db, s3: s3, log: log} +} + +func (r *Reconciler) note(rep *Report, it Item) { + if len(rep.Items) < 200 { + rep.Items = append(rep.Items, it) + } else { + rep.Truncated = true + } +} + +func (r *Reconciler) Run(ctx context.Context, opt Options) (*Report, error) { + if opt.Now == nil { + opt.Now = time.Now + } + unlock, ok, err := r.db.TryLock(ctx, lockID) + if err != nil { + return nil, err + } + if !ok { + return nil, ErrLocked + } + defer unlock() + + rep := &Report{Mode: "dry-run", GraceSeconds: int64(opt.Grace.Seconds()), Items: []Item{}} + if opt.Apply { + rep.Mode = "apply" + } + start := time.Now() + cutoff := opt.Now().Add(-opt.Grace) + + objects := make(map[string]objstore.Object) + if err := r.s3.List(ctx, Prefix, func(o objstore.Object) error { + objects[o.Key] = o + return nil + }); err != nil { + return nil, err + } + rep.ObjectsScanned = len(objects) + rep.ListMS = time.Since(start).Milliseconds() + + dbStart := time.Now() + known := make(map[string]bool, len(objects)) + var refs []store.AttachmentRef + if err := r.db.EachAttachmentRef(ctx, func(a store.AttachmentRef) error { + known[a.ObjectKey] = true + rep.RowsScanned++ + obj, present := objects[a.ObjectKey] + switch { + case a.Status == "pending" && a.CreatedAt.Before(cutoff): + refs = append(refs, a) + case a.Status == "ready" && !present && !a.Missing && a.CreatedAt.Before(cutoff): + refs = append(refs, a) + case a.Status == "ready" && present && a.Missing: + refs = append(refs, a) + case a.Status == "ready" && present && obj.Size != a.SizeBytes: + rep.SizeMismatch++ + r.note(rep, Item{Kind: "size_mismatch", Key: a.ObjectKey, ID: a.ID, Action: "report only"}) + } + return nil + }); err != nil { + return nil, err + } + rep.DBMS = time.Since(dbStart).Milliseconds() + + for _, a := range refs { + _, present := objects[a.ObjectKey] + switch { + case a.Status == "pending": + r.stalePending(ctx, rep, a, opt.Apply) + case !present: + r.missing(ctx, rep, a, opt.Apply) + default: + r.recovered(ctx, rep, a, opt.Apply) + } + } + + for key, o := range objects { + if known[key] { + continue + } + if !o.LastModified.Before(cutoff) { + rep.FreshSkipped++ + r.note(rep, Item{Kind: "orphan_object", Key: key, Action: "skip: younger than grace"}) + continue + } + r.orphan(ctx, rep, key, opt.Apply) + } + + rep.TotalMS = time.Since(start).Milliseconds() + if rep.ObjectsScanned > 0 { + rep.PerObjectUS = float64(time.Since(start).Microseconds()) / float64(rep.ObjectsScanned) + } + return rep, nil +} + +func (r *Reconciler) fail(rep *Report, it Item, err error) { + rep.Errors++ + it.Action = "error: " + strings.SplitN(err.Error(), "\n", 2)[0] + r.note(rep, it) +} + +// Stale pending row: the upload never reached "ready", so no message can +// reference it. The row is deleted first (conditionally), then the object. +func (r *Reconciler) stalePending(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) { + rep.StalePending++ + it := Item{Kind: "stale_pending", Key: a.ObjectKey, ID: a.ID, Action: "would delete row and object"} + if !apply { + r.note(rep, it) + return + } + deleted, err := r.db.DeletePendingAttachment(ctx, a.ID) + if err != nil { + r.fail(rep, it, err) + return + } + if !deleted { + it.Action = "skip: row changed concurrently" + r.note(rep, it) + return + } + if err := r.s3.Remove(ctx, a.ObjectKey); err != nil { + // The object becomes an orphan and is removed by the next run. + r.fail(rep, it, err) + return + } + rep.Fixed++ + it.Action = "deleted row and object" + r.note(rep, it) +} + +// Orphan object: no metadata row. The DB is re-checked right before deletion. +func (r *Reconciler) orphan(ctx context.Context, rep *Report, key string, apply bool) { + rep.OrphanObjects++ + it := Item{Kind: "orphan_object", Key: key, Action: "would delete object"} + if !apply { + r.note(rep, it) + return + } + known, err := r.db.ObjectKeyKnown(ctx, key) + if err != nil { + r.fail(rep, it, err) + return + } + if known { + it.Action = "skip: row appeared concurrently" + r.note(rep, it) + return + } + if err := r.s3.Remove(ctx, key); err != nil { + r.fail(rep, it, err) + return + } + rep.Fixed++ + it.Action = "deleted object" + r.note(rep, it) +} + +// Missing object of a ready row: bytes cannot be restored, metadata of a +// possibly referenced attachment is never deleted. The row is marked so that +// operators see it; StatObject confirms the absence first. +func (r *Reconciler) missing(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) { + _, err := r.s3.Stat(ctx, a.ObjectKey) + if err == nil { + return + } + it := Item{Kind: "missing_object", Key: a.ObjectKey, ID: a.ID, Action: "would mark missing (metadata kept)"} + if !objstore.IsNotFound(err) { + r.fail(rep, it, err) + return + } + rep.MissingObjects++ + if !apply { + r.note(rep, it) + return + } + if err := r.db.SetAttachmentMissing(ctx, a.ID, true); err != nil { + r.fail(rep, it, err) + return + } + rep.Fixed++ + it.Action = "marked missing (metadata kept)" + r.note(rep, it) +} + +func (r *Reconciler) recovered(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) { + rep.Recovered++ + it := Item{Kind: "recovered_object", Key: a.ObjectKey, ID: a.ID, Action: "would clear missing mark"} + if !apply { + r.note(rep, it) + return + } + if err := r.db.SetAttachmentMissing(ctx, a.ID, false); err != nil { + r.fail(rep, it, err) + return + } + rep.Fixed++ + it.Action = "cleared missing mark" + r.note(rep, it) +} diff --git a/src/internal/store/attachments.go b/src/internal/store/attachments.go new file mode 100644 index 00000000..5688e781 --- /dev/null +++ b/src/internal/store/attachments.go @@ -0,0 +1,152 @@ +package store + +import ( + "context" + "errors" + "time" + + "github.com/jackc/pgx/v5" + + "mtusi-messenger/internal/model" +) + +const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.object_key, a.missing_at IS NOT NULL, a.created_at, a.updated_at` + +func scanAttachment(row pgx.Row) (*model.Attachment, error) { + a := &model.Attachment{Status: "ready", Variants: []model.Variant{}} + var created, updated time.Time + err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, + &a.ObjectKey, &a.Missing, &created, &updated) + if err != nil { + return nil, err + } + a.CreatedAt, a.UpdatedAt = utc(created), utc(updated) + return a, nil +} + +// InsertPendingAttachment records the intent before the S3 PUT. A pending row is +// invisible to the API; if the upload never completes, the reconciler removes +// the row and the object after the grace period. +func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int) error { + var created time.Time + err := s.pool.QueryRow(ctx, ` + INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, status) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'pending') + RETURNING created_at`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, + width, height, a.ObjectKey).Scan(&created) + if pgCode(err) == "23503" { + return ErrNotFound + } + if err != nil { + return err + } + a.CreatedAt = utc(created) + return nil +} + +// MarkAttachmentReady succeeds only while the row is still pending: if the +// reconciler already removed a stale intent, the upload must not report 201. +func (s *Store) MarkAttachmentReady(ctx context.Context, id string) (time.Time, error) { + var updated time.Time + err := s.pool.QueryRow(ctx, ` + UPDATE attachments SET status = 'ready', updated_at = date_trunc('milliseconds', now()) + WHERE id = $1 AND status = 'pending' + RETURNING updated_at`, id).Scan(&updated) + if errors.Is(err, pgx.ErrNoRows) { + return time.Time{}, ErrNotFound + } + if err != nil { + return time.Time{}, err + } + return updated.UTC(), nil +} + +// DeletePendingAttachment removes an intent row that never became ready. +func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) { + tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +// AttachmentForUser returns a ready attachment only to a member of its chat. +func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) { + a, err := scanAttachment(s.pool.QueryRow(ctx, ` + SELECT `+attachmentColumns+` + FROM attachments a + WHERE a.id = $1 AND a.status = 'ready' + AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`, + id, userID)) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return a, err +} + +// AttachmentRef is the reconciler's view of one metadata row. +type AttachmentRef struct { + ID string + ObjectKey string + Status string + SizeBytes int64 + Missing bool + CreatedAt time.Time + Referenced bool +} + +func (s *Store) EachAttachmentRef(ctx context.Context, fn func(AttachmentRef) error) error { + rows, err := s.pool.Query(ctx, ` + SELECT a.id, a.object_key, a.status, a.size_bytes, a.missing_at IS NOT NULL, a.created_at, + EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) + FROM attachments a`) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var r AttachmentRef + if err := rows.Scan(&r.ID, &r.ObjectKey, &r.Status, &r.SizeBytes, &r.Missing, &r.CreatedAt, &r.Referenced); err != nil { + return err + } + if err := fn(r); err != nil { + return err + } + } + return rows.Err() +} + +func (s *Store) ObjectKeyKnown(ctx context.Context, key string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`, key).Scan(&ok) + return ok, err +} + +func (s *Store) SetAttachmentMissing(ctx context.Context, id string, missing bool) error { + _, err := s.pool.Exec(ctx, ` + UPDATE attachments + SET missing_at = CASE WHEN $2 THEN COALESCE(missing_at, now()) END + WHERE id = $1 AND status = 'ready'`, id, missing) + return err +} + +// TryLock takes a session-level advisory lock on a dedicated connection so that +// only one reconciler pass runs at a time. The returned func releases it. +func (s *Store) TryLock(ctx context.Context, id int64) (func(), bool, error) { + conn, err := s.pool.Acquire(ctx) + if err != nil { + return nil, false, err + } + var ok bool + if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock($1)`, id).Scan(&ok); err != nil || !ok { + conn.Release() + return nil, false, err + } + return func() { + _, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, id) + conn.Release() + }, true, nil +} diff --git a/src/internal/store/cursor.go b/src/internal/store/cursor.go new file mode 100644 index 00000000..209405b2 --- /dev/null +++ b/src/internal/store/cursor.go @@ -0,0 +1,33 @@ +package store + +import ( + "encoding/base64" + "encoding/json" + "time" +) + +type Cursor struct { + Scope string `json:"s"` + TS time.Time `json:"t"` + ID string `json:"i"` +} + +func EncodeCursor(c Cursor) string { + b, _ := json.Marshal(c) + return base64.RawURLEncoding.EncodeToString(b) +} + +func DecodeCursor(s string) (Cursor, error) { + var c Cursor + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return c, ErrInvalidCursor + } + if err := json.Unmarshal(b, &c); err != nil { + return c, ErrInvalidCursor + } + if c.Scope == "" || c.ID == "" || c.TS.IsZero() { + return c, ErrInvalidCursor + } + return c, nil +} diff --git a/src/internal/store/postgres.go b/src/internal/store/postgres.go new file mode 100644 index 00000000..e13c3c25 --- /dev/null +++ b/src/internal/store/postgres.go @@ -0,0 +1,385 @@ +package store + +import ( + "bytes" + "context" + "errors" + "net" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi-messenger/internal/model" +) + +var ( + ErrNotFound = errors.New("not found") + ErrUsernameTaken = errors.New("username taken") + ErrUnknownMember = errors.New("unknown member") + ErrInvalidCursor = errors.New("invalid cursor") +) + +type Store struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} } + +func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) } + +func IsUnavailable(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) { + return true + } + var connErr *pgconn.ConnectError + if errors.As(err, &connErr) { + return true + } + var netErr net.Error + if errors.As(err, &netErr) { + return true + } + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + + switch { + case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03", + len(pgErr.Code) == 5 && pgErr.Code[:2] == "08": + return true + } + } + return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed) +} + +func pgCode(err error) string { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) { + return pgErr.Code + } + return "" +} + +func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) } + +func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) { + u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName} + var created time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) RETURNING created_at`, + u.ID, username, displayName, passwordHash).Scan(&created) + if pgCode(err) == "23505" { + return nil, ErrUsernameTaken + } + if err != nil { + return nil, err + } + u.CreatedAt = utc(created) + return u, nil +} + +func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) { + u := &model.User{} + var created time.Time + var hash string + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`, + username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash) + if errors.Is(err, pgx.ErrNoRows) { + return nil, "", ErrNotFound + } + if err != nil { + return nil, "", err + } + u.CreatedAt = utc(created) + return u, hash, nil +} + +func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) { + var expires time.Time + err := s.pool.QueryRow(ctx, + `INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3)) + RETURNING expires_at`, + tokenHash, userID, ttl.Seconds()).Scan(&expires) + if err != nil { + return time.Time{}, err + } + return expires.UTC(), nil +} + +func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) { + u := &model.User{} + var created, expires time.Time + err := s.pool.QueryRow(ctx, + `SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`, + tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires) + if errors.Is(err, pgx.ErrNoRows) { + return nil, time.Time{}, ErrNotFound + } + if err != nil { + return nil, time.Time{}, err + } + u.CreatedAt = utc(created) + return u, expires.UTC(), nil +} + +func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error { + tag, err := s.pool.Exec(ctx, + `UPDATE sessions SET revoked_at = now() + WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, + `DELETE FROM sessions + WHERE expires_at < now() - make_interval(secs => $1) + OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) { + c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)} + err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error { + var created time.Time + if err := tx.QueryRow(ctx, + `INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`, + c.ID, title, creatorID).Scan(&created); err != nil { + return err + } + c.CreatedAt = utc(created) + _, err := tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, m.user_id, m.ord - 1 + FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`, + c.ID, memberIDs) + return err + }) + if pgCode(err) == "23503" { + return nil, ErrUnknownMember + } + if err != nil { + return nil, err + } + return c, nil +} + +func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + return ok, err +} + +func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) { + c := &model.Chat{} + var created time.Time + err := s.pool.QueryRow(ctx, + `SELECT c.id, c.title, c.created_at, + array_agg(m.user_id::text ORDER BY m.position) + FROM chats c JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = $1 + AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2) + GROUP BY c.id`, + chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, err + } + c.CreatedAt = utc(created) + return c, nil +} + +func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) { + var ts *time.Time + var cid *string + if cur != nil { + ts, cid = &cur.TS, &cur.ID + } + rows, err := s.pool.Query(ctx, + `SELECT c.id, c.title, c.created_at, + (SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) + FROM chat_members me JOIN chats c ON c.id = me.chat_id + WHERE me.user_id = $1 + AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid)) + ORDER BY c.created_at, c.id + LIMIT $4`, + userID, ts, cid, limit+1) + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Chat, 0, limit+1) + for rows.Next() { + c := &model.Chat{} + var created time.Time + if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil { + return nil, false, err + } + c.CreatedAt = utc(created) + out = append(out, c) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} + +var ( + ErrIdempotencyConflict = errors.New("idempotency key reused with different body") + ErrAttachmentNotFound = errors.New("attachment not found in chat") +) + +// AddMessage inserts the message, its idempotency record and attachment links in +// one statement. Attachments must be ready and belong to the same chat; the +// composite FK on message_attachments enforces the chat boundary in the DB too. +func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) { + for attempt := 0; attempt < 2; attempt++ { + newID := uuid.NewString() + var member, attachmentsOK bool + var created *time.Time + err := s.pool.QueryRow(ctx, ` + WITH member AS ( + SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1 + ), att AS ( + SELECT count(*) = cardinality($8::uuid[]) AS ok + FROM attachments + WHERE chat_id = $2 AND id = ANY($8::uuid[]) AND status = 'ready' + ), k AS ( + INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id) + SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok + ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now() + WHERE message_idempotency.created_at < now() - make_interval(secs => $7) + RETURNING message_id + ), m AS ( + INSERT INTO messages (id, chat_id, sender_id, text) + SELECT $5, $2, $1, $6 FROM k + RETURNING created_at + ), ma AS ( + INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + SELECT $5, $2, a.id, a.ord - 1 + FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord) + WHERE EXISTS (SELECT 1 FROM m) + ) + SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT created_at FROM m)`, + senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &created) + if pgCode(err) == "23503" { + return nil, false, ErrAttachmentNotFound + } + if err != nil { + return nil, false, err + } + if !member { + return nil, false, ErrNotFound + } + if !attachmentsOK { + return nil, false, ErrAttachmentNotFound + } + if created != nil { + return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text, + CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil + } + var stored []byte + var at time.Time + m := &model.Message{ChatID: chatID} + err = s.pool.QueryRow(ctx, ` + SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at, + `+attachmentIDsSQL+` + FROM message_idempotency i JOIN messages m ON m.id = i.message_id + WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3 + AND i.created_at >= now() - make_interval(secs => $4)`, + senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return nil, false, err + } + if !bytes.Equal(stored, reqHash) { + return nil, false, ErrIdempotencyConflict + } + m.CreatedAt = utc(at) + return m, true, nil + } + return nil, false, errors.New("idempotency record kept changing") +} + +const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position) + FROM message_attachments ma WHERE ma.message_id = m.id), '{}')` + +func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) { + tag, err := s.pool.Exec(ctx, + `DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) { + var ( + rows pgx.Rows + err error + ) + if cur == nil { + rows, err = s.pool.Query(ctx, + `SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+` + FROM messages m + WHERE m.chat_id = $1 + ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1) + } else { + rows, err = s.pool.Query(ctx, + `SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+` + FROM messages m + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid) + ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1) + } + if err != nil { + return nil, false, err + } + defer rows.Close() + out := make([]*model.Message, 0, limit+1) + for rows.Next() { + m := &model.Message{} + var created time.Time + if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil { + return nil, false, err + } + m.CreatedAt = utc(created) + out = append(out, m) + } + if err := rows.Err(); err != nil { + return nil, false, err + } + more := len(out) > limit + if more { + out = out[:limit] + } + return out, more, nil +} diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..b264f91a Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..d302a92a --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,34 @@ +"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git).""" +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, raw=None, level=6, color_type=2): + """RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise).""" + if raw is None: + row = 1 + 3 * width + raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height)) + ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + + chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b'')) + + +def near_limit_png(limit=10 * 1024 * 1024): + """Valid noise PNG a little below the byte limit (stored deflate blocks).""" + width = 1000 + height = (limit - 64 * 1024) // (1 + 3 * width) + data = png(width, height, level=0) + assert len(data) <= limit, len(data) + return data + + +def bomb_png(width=20000, height=20000): + """Header declares 400M pixels, the compressed payload is tiny.""" + return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + + chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b'')) diff --git a/student-tests/run_tests.py b/student-tests/run_tests.py new file mode 100644 index 00000000..87179168 --- /dev/null +++ b/student-tests/run_tests.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import threading +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, Failure, check + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import test_attachments + +RESULTS = [] + + +def case(name): + def wrap(fn): + def run(*a): + try: + detail = fn(*a) + RESULTS.append({'test': name, 'result': 'PASS', 'detail': detail}) + print('PASS', name, '-', detail or '') + except Failure as exc: + RESULTS.append({'test': name, 'result': 'FAIL', 'detail': str(exc)}) + print('FAIL', name, '-', exc) + return run + return wrap + + +def msg_path(chat): + return '/api/v1/chats/' + chat['id'] + '/messages' + + +def history(c, actor, chat): + out, cursor = [], None + while True: + q = '?limit=100' + ('&cursor=' + cursor if cursor else '') + page = c.call('GET', msg_path(chat) + q, actor=actor, schema='MessagePage')['json'] + out += page['items'] + cursor = page['next_cursor'] + if not cursor: + return out + + +@case('one session served by >=2 replicas without re-login (no stickiness)') +def t_no_sticky(c, alice, bob, chat): + seen = {} + for _ in range(30): + iid = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] + seen[iid] = seen.get(iid, 0) + 1 + check(len(seen) >= 2, 'only one replica observed: %s' % seen) + return 'distribution %s' % sorted(seen.values()) + + +@case('X-Instance-Id is present on errors and health too') +def t_instance_on_errors(c, alice, bob, chat): + r = c.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice) + check(bool(r['headers'].get('X-Instance-Id')), 'missing on 400') + r = c.call('GET', '/api/v1/users/me', 401) + check(bool(r['headers'].get('X-Instance-Id')), 'missing on 401') + r = c.call('GET', '/health/ready', schema='Health') + check(bool(r['headers'].get('X-Instance-Id')), 'missing on health') + + +@case('Idempotency-Key missing or malformed -> 400, nothing stored') +def t_key_validation(c, alice, bob, chat): + before = len(history(c, alice, chat)) + c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'без ключа'}) + for bad in ('', 'a b', 'clé', 'x' * 129, 'a/b'): + c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'плохой ключ'}, + headers={'Idempotency-Key': bad}) + c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'граница ключа'}, + headers={'Idempotency-Key': 'A-z0_9.:' + 'k' * 120}) + check(len(history(c, alice, chat)) == before + 1, 'rejected requests created messages') + + +@case('concurrent replay of one key through different replicas -> exactly one message') +def t_concurrent_same_key(c, alice, bob, chat): + key = 'conc-' + uuid.uuid4().hex + text = 'Конкурентный повтор ' + key + n = 24 + barrier = threading.Barrier(n) + raw = json.dumps({'text': text}, ensure_ascii=False).encode() + + def send(_): + local = Client(c.base, timeout=10) + barrier.wait() + r = local.call('POST', msg_path(chat), 201, actor=alice, raw=raw, + headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message') + return r['json'], r['headers']['X-Instance-Id'] + + with ThreadPoolExecutor(max_workers=n) as pool: + results = list(pool.map(send, range(n))) + bodies = {json.dumps(m, sort_keys=True) for m, _ in results} + instances = {i for _, i in results} + check(len(bodies) == 1, 'replays returned different messages: %d' % len(bodies)) + check(len(instances) >= 2, 'all requests hit one replica; test does not prove cross-replica') + same = [m for m in history(c, bob, chat) if m['text'] == text] + check(len(same) == 1, 'expected exactly 1 stored message, found %d' % len(same)) + return '%d parallel requests via %d replicas -> 1 message' % (n, len(instances)) + + +@case('replay is semantic: JSON key order/whitespace does not matter; other body -> 409') +def t_semantic_replay(c, alice, bob, chat): + key = uuid.uuid4().hex + first = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{"text":"abc"}', + headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message') + again = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{ "text" : "abc" }\n', + headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message') + check(first['json'] == again['json'], 'replay returned another message') + c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'abd'}, headers={'Idempotency-Key': key}) + third = c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'abc'}, + headers={'Idempotency-Key': key}, schema='Message') + check(third['json'] == first['json'], 'stored result changed after 409') + + +@case('key scope = user + chat: same key is independent for other user / other chat') +def t_key_scope(c, alice, bob, chat): + key = 'shared-' + uuid.uuid4().hex + a = c.message(alice, chat, 'scope', key=key)['json'] + b = c.message(bob, chat, 'scope', key=key)['json'] + chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Второй', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + a2 = c.message(alice, chat2, 'scope', key=key)['json'] + check(len({a['id'], b['id'], a2['id']}) == 3, 'scopes collided') + check(b['sender_id'] == bob['id'], 'sender mixed up') + + +@case('replay re-checks the session and chat access') +def t_replay_rechecks_access(c, alice, bob, chat, eve): + key = uuid.uuid4().hex + c.message(alice, chat, 'access', key=key) + c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'access'}, headers={'Idempotency-Key': key}) + c.call('POST', msg_path(chat), 401, body={'text': 'access'}, headers={'Idempotency-Key': key}) + c.call('POST', msg_path(chat), 404, actor=eve, body={'text': ' '}, headers={'Idempotency-Key': key}) + c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'x'}) + + +@case('logout is visible on every replica immediately (shared sessions)') +def t_logout_shared(c, alice, bob, chat): + tmp = c.user('tmp') + c.call('GET', '/api/v1/users/me', actor=tmp, schema='User') + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=tmp) + seen = set() + for _ in range(10): + seen.add(c.call('GET', '/api/v1/users/me', 401, actor=tmp)['headers']['X-Instance-Id']) + check(len(seen) >= 2, 'revocation checked on one replica only') + return 'revoked token rejected by %d replicas' % len(seen) + + +def main(): + import argparse + p = argparse.ArgumentParser() + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--json', default=None) + a = p.parse_args() + c = Client(a.base_url, timeout=60) + alice, bob, eve, chat = c.bootstrap() + for t in (t_no_sticky, t_instance_on_errors, t_key_validation, t_concurrent_same_key, + t_semantic_replay, t_key_scope, t_logout_shared): + t(c, alice, bob, chat) + t_replay_rechecks_access(c, alice, bob, chat, eve) + for needs_eve, t in test_attachments.register(case, history, msg_path): + t(c, alice, bob, chat, *([eve] if needs_eve else [])) + failed = [r for r in RESULTS if r['result'] != 'PASS'] + print('%d/%d passed' % (len(RESULTS) - len(failed), len(RESULTS))) + if a.json: + Path(a.json).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + sys.exit(1 if failed else 0) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..7b44c3dd --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,181 @@ +"""Lab4 attachment cases; registered by run_tests.py (same case() harness).""" +from concurrent.futures import ThreadPoolExecutor +import hashlib +import os +from pathlib import Path +import threading +import uuid +from urllib.parse import quote + +from client import Client, check +import imagegen + +FIXTURES = Path(__file__).resolve().parent / 'fixtures' +PNG = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() +JPG = (FIXTURES / 'sample.jpg').read_bytes() + + +def register(case, history, msg_path): + """Return the list of (needs_eve, fn) cases bound to the runner's helpers.""" + + def upload(c, actor, chat, data, ctype='image/png', name='sample.png', expected=201): + r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(name, safe=''), + expected, actor=actor, raw=data, headers={'Content-Type': ctype}, + schema='Attachment' if expected == 201 else None) + return r['json'] + + @case('PNG and JPEG upload/download byte-exact with true Content-Type') + def t_upload_formats(c, alice, bob, chat): + for data, ctype, name in ((PNG, 'image/png', 'a.png'), (JPG, 'image/jpeg', 'b.jpg')): + att = upload(c, alice, chat, data, ctype, name) + check(att['content_type'] == ctype and att['size_bytes'] == len(data), 'metadata mismatch') + check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'sha256 mismatch') + check(att['status'] == 'ready' and att['variants'] == [] and att['error_code'] is None, 'lab4 state') + meta = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json'] + check(meta == att, 'GET metadata differs from POST') + got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob) + check(got['bytes'] == data and got['headers'].get_content_type() == ctype, 'download differs') + check(got['headers'].get('X-Content-Type-Options') == 'nosniff', 'nosniff missing') + return 'png %d B, jpeg %d B' % (len(PNG), len(JPG)) + + @case('size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413') + def t_size_limit(c, alice, bob, chat): + near = imagegen.near_limit_png() + att = upload(c, alice, chat, near, name='near-limit.png') + got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=alice) + check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'] == hashlib.sha256(near).hexdigest(), + 'big file changed') + over = PNG + b'\0' * (10 * 1024 * 1024 + 1 - len(PNG)) + upload(c, alice, chat, over, name='over.png', expected=413) + return 'accepted %d B, rejected %d B' % (len(near), len(over)) + + @case('decompression bomb (400M px header, tiny body) -> 413 before decoding') + def t_bomb(c, alice, bob, chat): + data = imagegen.bomb_png() + r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bomb.png', 413, + actor=alice, raw=data, headers={'Content-Type': 'image/png'}) + check(r['elapsed'] < 2, 'bomb took %.2fs: was it decoded?' % r['elapsed']) + return '%d B rejected in %.0f ms' % (len(data), r['elapsed'] * 1000) + + @case('corrupted images and MIME spoofing -> 415, never 500') + def t_corrupt_and_spoof(c, alice, bob, chat): + cases = { + 'truncated png': (PNG[:len(PNG) // 2], 'image/png'), + 'garbage after png header': (PNG[:40] + os.urandom(2000), 'image/png'), + 'truncated jpeg': (JPG[:len(JPG) // 2], 'image/jpeg'), + 'png declared as jpeg': (PNG, 'image/jpeg'), + 'jpeg declared as png': (JPG, 'image/png'), + 'gif bytes as png': (b'GIF89a' + os.urandom(100), 'image/png'), + 'html as png': (b'', 'image/png'), + 'text/plain content type': (PNG, 'text/plain'), + 'multipart content type': (PNG, 'multipart/form-data; boundary=x'), + } + for name, (data, ctype) in cases.items(): + r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=x.png', 415, + actor=alice, raw=data, headers={'Content-Type': ctype}) + check(r['json']['error']['code'] == 'unsupported_media_type', name) + upload(c, alice, chat, b'', name='empty.png', expected=400) + return '%d spoof/corrupt variants -> 415' % len(cases) + + @case('dangerous filenames -> 400; Unicode display name kept as-is') + def t_filenames(c, alice, bob, chat): + bad = ['../etc/passwd', 'a/b.png', 'a\\b.png', 'nul\x00.png', 'bell\x07.png', 'line\nbreak.png', + '', ' ', '.', '..', 'x' * 256, 'evil‮gnp.exe'] + for name in bad: + upload(c, alice, chat, PNG, name=name, expected=400) + base = '/api/v1/chats/' + chat['id'] + '/attachments' + c.call('POST', base, 400, actor=alice, raw=PNG, headers={'Content-Type': 'image/png'}) + c.call('POST', base + '?filename=a.png&filename=b.png', 400, actor=alice, raw=PNG, + headers={'Content-Type': 'image/png'}) + nice = 'Фото кота 🐱 (1).png' + att = upload(c, alice, chat, PNG, name=nice) + check(att['filename'] == nice, 'filename changed') + long_ok = upload(c, alice, chat, PNG, name='я' * 255) + check(len(long_ok['filename']) == 255, '255 code points must pass') + return '%d bad names rejected' % len(bad) + + @case('ACL: outsider gets 404 on upload, metadata, content; no token -> 401') + def t_attachment_acl(c, alice, bob, chat, eve): + att = upload(c, alice, chat, PNG) + upload(c, eve, chat, PNG, expected=404) + for suffix in ('', '/content', '/content?variant=original'): + c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), 404, actor=alice) + c.call('GET', '/api/v1/attachments/not-a-uuid', 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=alice) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=huge', 400, actor=alice) + + @case('message attachment_ids: other chat / foreign / unknown -> 404, nothing stored') + def t_message_links(c, alice, bob, chat, eve): + # alice is a member of both chats: the file still must not cross the chat boundary + chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + other = upload(c, alice, chat2, PNG) + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, + body={'title': 'Чат Евы', 'member_ids': [eve['id']]}, schema='Chat')['json'] + foreign = upload(c, eve, eve_chat, PNG) + own = upload(c, alice, chat, PNG) + before = len(history(c, alice, chat)) + for ids in ([other['id']], [foreign['id']], [str(uuid.uuid4())], [own['id'], other['id']]): + r = c.call('POST', msg_path(chat), 404, actor=alice, body={'text': 'x', 'attachment_ids': ids}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(r['json']['error']['code'] == 'not_found', 'expected not_found') + check(len(history(c, alice, chat)) == before, 'rejected message was stored') + m = c.message(bob, chat, '', attachments=[own['id']])['json'] + check(m['attachment_ids'] == [own['id']] and m['text'] == '', 'bob could not reference chat file') + listed = [x for x in history(c, alice, chat) if x['id'] == m['id']] + check(listed == [m], 'history lost attachment_ids') + + @case('attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400') + def t_message_validation(c, alice, bob, chat): + att = upload(c, alice, chat, PNG) + for body in ({'text': 'x', 'attachment_ids': [att['id'], att['id']]}, + {'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]}, + {'text': 'x', 'attachment_ids': ['nope']}, + {'text': 'x', 'attachment_ids': 'nope'}, + {'text': '', 'attachment_ids': []}, + {'text': ' '}): + c.call('POST', msg_path(chat), 400, actor=alice, body=body, + headers={'Idempotency-Key': uuid.uuid4().hex}) + ten = [upload(c, alice, chat, PNG)['id'] for _ in range(10)] + m = c.message(alice, chat, 'десять файлов', attachments=ten)['json'] + check(m['attachment_ids'] == ten, 'order of attachment_ids changed') + + @case('idempotency: missing attachment_ids == []; other attachment list -> 409') + def t_attachment_idempotency(c, alice, bob, chat): + key = uuid.uuid4().hex + first = c.message(alice, chat, 'idem', key=key)['json'] + again = c.message(alice, chat, 'idem', key=key, attachments=[])['json'] + check(first == again and first['attachment_ids'] == [], 'missing vs [] not equal') + att = upload(c, alice, chat, PNG) + c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'idem', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': key}) + key2 = uuid.uuid4().hex + a = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json'] + b = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json'] + check(a == b, 'replay with attachments differs') + + @case('10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back') + def t_concurrent_uploads(c, alice, bob, chat): + n = 10 + barrier = threading.Barrier(n) + + def one(i): + local = Client(c.base, timeout=60) + barrier.wait() + r = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=p%d.png' % i, 201, + actor=alice, raw=PNG, headers={'Content-Type': 'image/png'}, schema='Attachment') + return r['json'], r['headers']['X-Instance-Id'] + + with ThreadPoolExecutor(max_workers=n) as pool: + res = list(pool.map(one, range(n))) + check(len({a['id'] for a, _ in res}) == n, 'ids collided') + for a, _ in res: + got = c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=bob) + check(got['bytes'] == PNG, 'content mixed up') + return '%d uploads via %d replicas' % (n, len({i for _, i in res})) + + return [(False, t_upload_formats), (False, t_size_limit), (False, t_bomb), (False, t_corrupt_and_spoof), + (False, t_filenames), (False, t_message_validation), (False, t_attachment_idempotency), + (False, t_concurrent_uploads), (True, t_attachment_acl), (True, t_message_links)]