forked from mtusi2026/mtusi2026lab4
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e094e13dc3 | ||
|
|
1706ddca25 |
+15
-2
@@ -7,14 +7,27 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Student implementation knobs (see REPORT.md)
|
||||
SESSION_CACHE_MAX_TTL_SECONDS=60
|
||||
DB_MAX_CONNS=10
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=512M
|
||||
API_GOMEMLIMIT=400MiB
|
||||
SHUTDOWN_DRAIN_MS=3000
|
||||
S3_TIMEOUT=15s
|
||||
UPLOAD_CONCURRENCY=8
|
||||
DECODE_BUDGET_BYTES=134217728
|
||||
UPLOAD_TIMEOUT=60s
|
||||
RECONCILE_GRACE=15m
|
||||
RECONCILE_EVERY=10m
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
FROM golang:1.23.4-alpine3.21 AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY src/go.mod src/go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY src/ ./
|
||||
ENV CGO_ENABLED=0
|
||||
RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
|
||||
|
||||
FROM alpine:3.21.2 AS runtime
|
||||
RUN addgroup -S app && adduser -S -G app app
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/server /app/server
|
||||
ARG APP_BUILD=dev
|
||||
ENV APP_BUILD=$APP_BUILD
|
||||
LABEL org.opencontainers.image.revision=$APP_BUILD
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
ENV APP_HOST=0.0.0.0 APP_PORT=8080
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
|
||||
ENTRYPOINT ["/app/server"]
|
||||
@@ -28,3 +28,40 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
# ---- student targets (lab4 solution) ----
|
||||
.PHONY: secrets up down ps restore test-student ops-checks memory track2-faults track2-cost
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
|
||||
up: secrets
|
||||
docker compose up -d --build --wait
|
||||
|
||||
down:
|
||||
docker compose stop
|
||||
|
||||
ps:
|
||||
docker compose ps -a
|
||||
|
||||
restore:
|
||||
docker compose up -d --wait
|
||||
|
||||
test-student:
|
||||
$(PYTHON) student-tests/run_tests.py --base-url "$(BASE_URL)" --json evidence/student-tests.json
|
||||
|
||||
ops-checks:
|
||||
$(PYTHON) scripts/lab4ops.py private --output evidence/private.json
|
||||
$(PYTHON) scripts/lab4ops.py failover-upload --output evidence/failover-upload.json
|
||||
$(PYTHON) scripts/lab4ops.py no-local-file --output evidence/no-local-file.json
|
||||
$(PYTHON) scripts/lab4ops.py bootstrap --output evidence/bootstrap.json
|
||||
$(PYTHON) scripts/lab4ops.py s3-outage --output evidence/s3-outage.json
|
||||
|
||||
memory:
|
||||
$(PYTHON) scripts/lab4ops.py memory --output evidence/memory.json
|
||||
|
||||
track2-faults:
|
||||
$(PYTHON) scripts/track2/run_track2.py faults --output evidence/track2/faults.json
|
||||
|
||||
track2-cost:
|
||||
$(PYTHON) scripts/track2/run_track2.py cost --output evidence/track2/cost.json
|
||||
|
||||
@@ -1,37 +1,183 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Кораблев Петр, БВТ2304
|
||||
- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 2 «Согласованность двух хранилищ»**)
|
||||
- Commit/PR: ветка `solution/BVT2304-korablev`
|
||||
- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, net/http, pgx/v5 5.7.2, go-redis/v9 9.7.0, golang.org/x/crypto (Argon2id), golang.org/x/sync 0.10.0 (semaphore), google/uuid 1.6.0, **minio-go/v7 7.0.82** (S3-клиент, работает с любым S3-совместимым сервером), стандартные `image/png` и `image/jpeg`. Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, **SeaweedFS 3.80** (`chrislusf/seaweedfs:3.80`), golang:1.23.4-alpine3.21 → alpine:3.21.2; для проверок вне API — python:3.12.8-alpine3.21. Документация: AWS S3 (uploads, SigV4, ListObjectsV2), SeaweedFS wiki (S3 API, `s3.config` identities), Go `image.DecodeConfig`. Помощь ИИ использовалась при написании отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже запускаются через `python`). На Windows тесты запускать с `PYTHONUTF8=1` (контракт в UTF-8).
|
||||
|
||||
```sh
|
||||
python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait # = make up
|
||||
make check && make test # smoke через LB http://localhost:8080
|
||||
make test-student # собственные тесты (18 кейсов)
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
|
||||
docker compose up -d --no-recreate --wait api # вернуть остановленную реплику
|
||||
make ops-checks memory track2-faults track2-cost # уровень 4 и трек 2 (нужен Docker)
|
||||
```
|
||||
|
||||
- `init_secrets.py` генерирует случайные пароли PostgreSQL/Valkey и **две S3-идентичности**: `bootstrap-admin` (только для `s3-init`) и `messenger-api` (права `Read/Write/List` только на bucket `messenger-attachments`), а также `secrets/s3_identities.json` — конфиг SeaweedFS. Всё в `secrets/` (в `.gitignore` и `.dockerignore`), контейнерам передаётся как Compose secrets `/run/secrets/*`.
|
||||
- Порядок старта задан `depends_on`: postgres/valkey/s3 healthy → `migrate` (миграции под advisory lock) → `s3-init` (идемпотентное создание bucket) → `api` ×2 → `lb`; `reconciler` стартует после `s3-init`. Готовность — `--wait`/`/health/ready`.
|
||||
- Безопасная остановка: `docker compose stop` (`make down`), данные в volumes `pgdata`, `s3data` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (удаляет volumes — необратимо).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
client → HAProxy (lb, 127.0.0.1:8080) → api ×2 (Go, read-only rootfs)
|
||||
├─ PostgreSQL: users, chats, messages, sessions, idempotency,
|
||||
│ attachments (метаданные), message_attachments
|
||||
├─ Valkey: кэш сессий (из lab3)
|
||||
└─ SeaweedFS S3 (bucket messenger-attachments, приватный): байты
|
||||
reconciler (тот же образ, `reconcile --apply --every 10m`) — сверка БД ↔ S3
|
||||
```
|
||||
|
||||
**Перенесено из lab3** без изменения поведения: пользователи, Basic → Bearer-сессии (Argon2id, Valkey-кэш), чаты, keyset-пагинация, обязательный `Idempotency-Key`, `X-Instance-Id`, HAProxy с health-check, хуки restart/stop-one. **Изменено:** убран экспериментальный путь `MESSAGE_WRITE_PATH=tx` трека 1 lab3 (остался один путь — one-shot CTE), лимит памяти API 256M → 512M, таймауты HAProxy client/server 30s/10s → 75s (загрузка до 10 MiB).
|
||||
|
||||
**Выбор S3-сервера.** Планировался MinIO, но на 2026-10 образы `minio/minio` и `minio/mc` недоступны ни в Docker Hub, ни в quay.io (pull access denied / 401). Выбран SeaweedFS 3.80: одна команда `weed server -s3` (master+volume+filer+S3), идентичности с правами на конкретный bucket через `-s3.config`, без анонимной идентичности — значит bucket приватный. S3-порт 8333 не публикуется, консоли master (9333) и filer (8888) тоже: сервис только в сети `data`.
|
||||
|
||||
**Модель данных** (`src/internal/db/migrations/0003_attachments.sql`):
|
||||
- `attachments`: `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key` (UNIQUE, генерирует сервер: `attachments/<chat_id>/<attachment_id>`), `status ∈ {pending, ready}`, `missing_at`, `created_at`, `updated_at`. FK `(chat_id, owner_id) → chat_members`, CHECK-и на размер ≤ 10 MiB, пиксели ≤ 40M, формат sha256.
|
||||
- `message_attachments(message_id, chat_id, attachment_id, position)` с составным FK `(chat_id, attachment_id) → attachments(chat_id, id)`: **сама БД** не даёт привязать к сообщению файл другого чата.
|
||||
- `messages.text` теперь может быть пустым (CHECK 0..4000); правило «пустой текст только при ≥1 вложении» проверяет API.
|
||||
|
||||
**Загрузка** `POST /api/v1/chats/{id}/attachments?filename=…` (`src/internal/api/attachments.go`):
|
||||
1. Аутентификация → членство в чате (посторонний получает 404 до чтения тела) → `filename` (1..255 code points, без `/`, `\`, NUL, управляющих и bidi-override символов, не `.`/`..`) → `Content-Type` ∈ {image/png, image/jpeg} иначе 415 → `Content-Length > 10 MiB` → 413 сразу.
|
||||
2. Слот загрузки (семафор `UPLOAD_CONCURRENCY=8` на реплику), тело читается через `http.MaxBytesReader(10 MiB)` → 413 при превышении. Тело буферизуется в памяти (≤ 10 MiB на слот), значит память одной реплики под загрузки ограничена 8 × 10 MiB.
|
||||
3. Формат определяется по magic bytes и сравнивается с заявленным (подмена → 415); `image.DecodeConfig` читает только заголовок: >40M пикселей → 413 **до** декодирования; затем полное декодирование под общим бюджетом памяти (`DECODE_BUDGET_BYTES=128 MiB`, взвешенный семафор по оценке `пиксели × байт/пиксель`) — повреждённые данные → 415.
|
||||
4. Транзакционная граница: **(a)** `INSERT attachments status='pending'` → **(b)** S3 `PUT` (Content-MD5) → **(c)** `UPDATE … SET status='ready' WHERE status='pending'` → `201 ready`. 201 возвращается только когда и байты, и метаданные зафиксированы.
|
||||
5. Ошибка на (b) или (c) → 503 и фоновая компенсация, которая начинается со строки: `DELETE … WHERE status='pending'`, и только если строка удалена — удаление объекта. Если `UPDATE → ready` на самом деле закоммитился, а ответ БД потерялся, строка уже ready, и объект не трогаем (иначе получилось бы ready-вложение без байтов). Если недоступна БД, остаётся stale pending; если недоступен S3, может остаться объект без строки. Оба случая убирает reconciler после grace.
|
||||
|
||||
**Скачивание** `GET /attachments/{id}/content`: только участники чата (иначе 404), только `status='ready'`; `Stat` до первого байта ответа (отсутствие объекта → 404, недоступность S3 → 503), затем поток из S3 в ответ, без redirect; заголовки `Content-Type` (истинный), `Content-Length`, `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `ETag: "<sha256>"`. `variant=thumbnail|crop|optimized` → 404 (вариантов в lab4 нет), неизвестный → 400.
|
||||
|
||||
**Сообщение** с `attachment_ids`: 0..10 уникальных UUID (иначе 400). Проверка «все ready и в этом чате», запись idempotency, сообщения и связей — **одним SQL-оператором** (CTE), как в lab3. Чужое, неизвестное или из другого чата → 404, ничего не сохраняется. Отпечаток идемпотентности считается по `{text, attachment_ids}` с нормализацией «нет поля» = `[]`.
|
||||
|
||||
**Готовность и отказы.** `/health/ready` зависит только от PostgreSQL. Недоступность S3 ломает только эндпоинты вложений (503), а текстовый мессенджер продолжает работать: если бы ready зависел от S3, LB снял бы обе реплики и упал бы весь API. SDK S3 настроен на быстрый отказ: `MaxRetry=1`, dial-таймаут 2 с, `ResponseHeaderTimeout` 10 с, операции под `S3_TIMEOUT=15s`.
|
||||
|
||||
**Соответствие `.env.example`** — все переменные реально используются в `compose.yaml`: `S3_ENDPOINT=http://s3:8333`, `S3_REGION`, `S3_BUCKET`, `S3_*_FILE` (secrets), `MAX_UPLOAD_BYTES` (не больше 10 MiB, иначе старт с ошибкой), `MAX_IMAGE_PIXELS`, плюс `UPLOAD_CONCURRENCY`, `DECODE_BUDGET_BYTES`, `UPLOAD_TIMEOUT`, `S3_TIMEOUT`, `API_MEMORY`, `API_GOMEMLIMIT`, `RECONCILE_GRACE`, `RECONCILE_EVERY`.
|
||||
|
||||
| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — |
|
||||
| API ×2 | api | mtusi-messenger:lab4 (сборка) | `/health/ready` | нет (read-only rootfs, tmpfs /tmp 1m) | app, data | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key |
|
||||
| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password |
|
||||
| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth |
|
||||
| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data, не публикуется | s3_identities.json |
|
||||
| Bootstrap | migrate, s3-init | mtusi-messenger:lab4 | one-shot | — | data | database_url / s3_admin_* |
|
||||
| Сверка | reconciler | mtusi-messenger:lab4 | — | — | data | database_url, s3 api-ключи |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Окружение всех замеров: Windows 11, Docker Desktop 29.7.2 / Compose 5.5.0, WSL2 kernel 6.18; лимиты: API 1 CPU / 512M ×2, S3 1 CPU / 1G, PostgreSQL 2 CPU / 1G, reconciler 0.5 CPU / 128M. Дата — 2026-10-29.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
### Уровень 3
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | `make check`, `make test` | PASS: lab4 public API scenario | evidence/check-smoke.txt |
|
||||
| Возможности lab3 сохранены | `make test-student` (8 кейсов lab3) | no-sticky, идемпотентность через 2 реплики, logout на всех репликах — PASS | evidence/student-tests.txt |
|
||||
| S3 в общем стенде, байты в приватном bucket | `lab4ops.py private` | аноним GET/LIST/CreateBucket → 403; api-ключ GET → 200, sha256 совпадает; порт S3 не опубликован | evidence/private.json |
|
||||
| Метаданные в PostgreSQL, не байты в БД/на диске API | `lab4ops.py no-local-file` | у API read-only rootfs (`touch` → Read-only file system), монтируются только secrets, новых файлов 0 | evidence/no-local-file.json |
|
||||
| 201 ready, побайтный download, ACL | smoke + `t_upload_formats`, `t_attachment_acl` | PNG и JPEG: sha256/размер/Content-Type совпадают; посторонний получает 404 на метаданные/content/upload, без токена 401 | evidence/student-tests.txt |
|
||||
| Реальные PNG/JPEG ≤10 MiB, ≤40M px, MIME по содержимому, опасные filename | `t_size_limit`, `t_bomb`, `t_corrupt_and_spoof`, `t_filenames` | PNG 10 420 335 B → 201; 10 MiB+1 → 413; заголовок 20000×20000 → 413 за 32 мс; 9 вариантов подмены/повреждения → 415; 12 опасных имён → 400 | evidence/student-tests.txt |
|
||||
| Upload → стоп обслужившей реплики → download через другую | `lab4ops.py failover-upload` | загрузка через `api-2`, `docker stop api-2`, 10/10 скачиваний через `api-1` с верным sha256 | evidence/failover-upload.json |
|
||||
| Failover (публичный тест) | `make test-failover` для каждой реплики | остановлена api-2: PASS, 123 записи, 0 ошибок, восстановление за 4.1 с; остановлена api-1: PASS, 116 записей, 0 ошибок, 4.1 с; файл цел в обоих случаях | evidence/failover.txt |
|
||||
| Restart всего стенда с volumes | `make test-persistence` (`restart.cmd`) | PASS; postgres/valkey/s3/api перезапущены (uptime до/после в файле), файл и сессия сохранились | evidence/persistence.txt |
|
||||
|
||||
### Уровень 4
|
||||
|
||||
| Критерий | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Тесты: большой файл, повреждённое изображение, подмена MIME, чужой attachment_id, файл другого чата; нет 500 | `make test-student` (10 кейсов lab4) | 18/18 PASS; в логах API за все прогоны нет ни одного 500 | evidence/student-tests.txt |
|
||||
| Нет вечных неучтённых объектов: компенсация + сверка с grace | pending-intent + компенсация + `reconciler` (grace 15 мин, каждые 10 мин) | см. трек 2; реальный случай: после падения S3 reconciler нашёл 19 stale pending из прерванных загрузок и удалил их | evidence/track2/faults.json |
|
||||
| Отказ S3 | `lab4ops.py s3-outage` | upload 503 за 2.0 с, download 503 за 2.0 с, метаданные 200, текстовое сообщение 201, ready 200; после старта S3 загрузки сразу работают, старый файл цел; pending-строк не осталось (компенсация удалила строку) | evidence/s3-outage.json |
|
||||
| Ограниченная RAM, ≥10 одновременных загрузок | `make memory` | см. «Эксперименты»: пик API ≤ 311 MiB при 40 одновременных 10-МБ загрузках, плато уже после 20 | evidence/memory.json |
|
||||
| Bootstrap идемпотентен | `lab4ops.py bootstrap` (двойной запуск `migrate` и `s3-init`) | «migrations up to date», `created:false` | evidence/bootstrap.json |
|
||||
| Отдельные ограниченные S3 credentials; консоль закрыта; private bucket проверен вне API | `lab4ops.py private` | api-идентичность: CreateBucket → 403, PUT в другой bucket → 403; S3/консоли не опубликованы; аноним → 403 | evidence/private.json |
|
||||
|
||||
### Трек 5.2 «Согласованность двух хранилищ»
|
||||
|
||||
| Критерий | Команда | Результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Повторяемый reconciler с dry-run | `server reconcile [--apply] [--grace] [--every]` | dry-run по умолчанию; advisory lock — одновременно идёт один проход | src/internal/reconcile/reconcile.go |
|
||||
| Ошибки на обеих сторонах: обнаружение и исправление | `make track2-faults` | 5 из 5 внедрённых сбоев найдены, 7 исправлений, повторный apply → 0 | evidence/track2/faults.json |
|
||||
| Свежие и используемые не удаляются | там же | свежий orphan, свежий pending и используемые U1/U2 остались целы | evidence/track2/faults.json |
|
||||
| Стоимость обхода | `make track2-cost` | линейно, ~70 мкс/объект, 20k за 1.4–1.6 с | evidence/track2/cost.json |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Память API при одновременных загрузках (уровень 4)
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Гипотеза:** пиковая память реплики ограничена `≈ база + UPLOAD_CONCURRENCY × 10 MiB + бюджет декодирования`, поэтому она выходит на плато и не растёт с числом клиентов.
|
||||
|
||||
**Методика:** `scripts/lab4ops.py memory`. Синтетический шумовой PNG 1000×3480, 10 420 335 B (почти лимит). Перед каждым прогоном `docker compose restart api`, одна прогревочная загрузка, затем N одновременных загрузок (барьер) через LB. Пик читается из cgroup v2 `memory.peak` каждой реплики. По 2 повтора.
|
||||
|
||||
| Одновременно | Успешно | Wall, с | p50 / max, мс | Пик API, MiB (max из 2 реплик) |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | 1/1, 1/1 | 0.35 | 337–344 | 64.4 / 66.4 |
|
||||
| 10 | 10/10, 10/10 | 3.1–4.1 | 2844–3830 / 2882–3945 | 148.2 / 151.7 |
|
||||
| 20 | 20/20, 20/20 | 6.4–7.1 | 5476–6097 / 6062–6830 | 286.0 / 263.7 |
|
||||
| 40 | 40/40, 40/40 | 11.8–12.6 | 8542–9590 / 11156–12044 | 264.5 / 311.1 |
|
||||
|
||||
**Вывод:** память выходит на плато ~250–310 MiB между 20 и 40 клиентами (слоты по 8 на реплику). Лишние клиенты ждут слота, это видно по росту latency, а не памяти. Это ниже лимита 512M и GOMEMLIMIT 400MiB. **Ограничения:** буфер одного тела ≤10 MiB в RAM — сознательное упрощение вместо потокового парсера PNG. 40M-пиксельный RGBA PNG при декодировании занимает ~160 MB и упирается в бюджет 128 MiB (декодируется один, остальные ждут). Запуск на одном ноутбуке, клиент и стенд делят CPU.
|
||||
|
||||
**Найденный отказ:** первый прогон шёл с лимитом S3 512M, и на 20 параллельных PUT процесс SeaweedFS перезапустился (RestartCount 1, логов завершения нет — kill по памяти). 18 из 20 загрузок получили корректный 503, их pending-строки остались (тогдашняя версия компенсации сначала удаляла объект и при недоступном S3 сохраняла строку) и затем были найдены reconciler как 19 stale pending, вместе с одной от теста отказа S3. Лимит S3 поднят до 1G: в повторе 0 рестартов при `memory.peak` ≈ 1 GiB. Этот пик включает page cache записываемых файлов, который ядро может вытеснить.
|
||||
|
||||
### Трек 2: внедрение ошибок (grace = 1 мин для эксперимента, 15 мин в стенде)
|
||||
|
||||
Методика `scripts/track2/run_track2.py faults`. Периодический reconciler на время теста останавливается. Сбои вносятся напрямую: в S3 через `scripts/track2/s3tool.py` (SigV4 на stdlib, запускается в одноразовом контейнере в сети `data`), в БД через `psql`.
|
||||
|
||||
| Сбой (сторона) | Ожидание | dry-run | apply |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| F1 объект без строки (S3) | удалить после grace | orphan_object | удалён |
|
||||
| F2 потеряна строка неиспользуемого U3 (БД) | объект стал сиротой → удалить | orphan_object | удалён |
|
||||
| F3 потерян объект используемого U2 (S3) | **метаданные не трогать**, пометить | missing_object | `missing_at` выставлен; GET метаданных 200, content 404, сообщение по-прежнему ссылается на U2 |
|
||||
| F4 падение между PUT и ready (pending + объект, 1 ч) | удалить строку, затем объект | stale_pending | удалены оба |
|
||||
| F5 падение до PUT (pending без объекта, 1 ч) | удалить строку | stale_pending | удалена |
|
||||
| F7 загрузка «в процессе» (свежие pending + объект) | не трогать | не найдена (row известен) | цела |
|
||||
| F8 свежий объект без строки | не трогать | `skip: younger than grace` | цел |
|
||||
| U1 используемый, исправный | не трогать | — | скачивается, sha256 верный |
|
||||
|
||||
- **Dry-run ничего не меняет:** счётчики S3-объектов, строк, pending и missing до и после совпадают (`dry_run_changed_nothing: true`).
|
||||
- **Первый apply:** 7 исправлений, 0 ошибок.
|
||||
- **Повторный apply:** 0 исправлений, только `skip` свежего F8 — проход идемпотентен.
|
||||
- **Восстановление:** после возврата объекта U2 «из бэкапа» (те же байты) apply выдаёт `recovered=1`, снимает `missing_at`, и скачивание отдаёт исходный sha256.
|
||||
- **Сторонние находки:** dry-run отдельно учитывает находки вне чата эксперимента (`other_findings_outside_experiment`) — это остатки предыдущих прогонов.
|
||||
|
||||
**Почему это безопасно.** Порядок «сначала LIST S3, потом скан БД» плюс «pending-строка пишется до PUT» означают, что у любого объекта из листинга строка уже есть. Перед каждым разрушающим действием есть повторная проверка: сирота — `ObjectKeyKnown` в БД, missing — `StatObject`, pending — `DELETE … WHERE status='pending'`. Удаляются только объекты без строк и pending-строки; ни одна из них не может быть в `message_attachments`, потому что привязать можно только ready. Используемые метаданные никогда не удаляются. Grace 15 мин намного больше бюджета загрузки (60 с). Если `UPDATE → ready` проигрывает гонку с reconciler, загрузка отвечает 503, а не 201.
|
||||
|
||||
### Трек 2: стоимость обхода
|
||||
|
||||
`scripts/track2/run_track2.py cost`: в БД добавляются N ready-строк (SQL `generate_series`), в S3 — N объектов по 100 B. Каждый размер замеряется трижды в dry-run, после замеров bench-данные удаляются.
|
||||
|
||||
| Объектов | list S3, мс | скан БД, мс | всего, мс | мкс/объект | пик памяти reconciler, MiB | находок |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| 1 229 | 52 | 6–9 | 59–61 | 48–50 | 10.1–10.9 | 0 |
|
||||
| 5 229 | 258–345 | 62–70 | 326–408 | 62–78 | 15.8–17.2 | 0 |
|
||||
| 20 229 | 1321–1537 | 41–95 | 1404–1618 | 69–80 | 30.8–31.2 | 0 |
|
||||
|
||||
**Вывод:** стоимость линейна, примерно 70 мкс и 1 КБ RAM на объект. Больше 90% времени занимает постраничный LIST в S3 (по 1000 ключей). На 20k объектов ложных срабатываний 0. При частоте раз в 10 минут нагрузка ничтожна. **Ограничение:** все ключи держатся в памяти, на 10M объектов это ~1 ГБ. Дальше нужен обход по префиксам `attachments/<chat_id>/` или потоковое слияние двух отсортированных списков: ключи S3 и `ORDER BY object_key` в БД.
|
||||
|
||||
### Сводка отказов
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| `docker stop` обслужившей API-реплики | LB уводит трафик | первая успешная запись через ~4 с, 0 дублей; файл скачивается через другую | без потерь |
|
||||
| Restart postgres+valkey+s3+api | данные, сессия и файл живы | PASS, тот же токен работает | без потерь |
|
||||
| Остановка S3 | 503 только на вложениях | upload/download 503 за 2 с, текст и ready работают | после старта S3 сразу; pending-строк не осталось |
|
||||
| SeaweedFS убит по памяти (лимит 512M) | 503, без 201 на несохранённое | 18× 503, 2× 201 | 19 pending-строк убраны reconciler; лимит поднят до 1G |
|
||||
| Внедрённые orphan/missing/pending | найти, исправить безопасно | 5/5 найдены, повторный apply = no-op | missing-байты невосстановимы без бэкапа: метаданные помечены и сохранены |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host, по одному PostgreSQL, S3-узлу и LB: выдерживается отказ одной API-реплики и рестарт стенда, но не потеря хоста или диска S3. В SeaweedFS нет репликации томов.
|
||||
- Тело загрузки буферизуется в RAM до 10 MiB на слот. Это ограничено и измерено, но не полностью потоковый режим.
|
||||
- Полное декодирование нужно для отсева повреждённых файлов и стоит CPU: 40 одновременных загрузок 10 МБ занимают ~12 с на 2×1 CPU.
|
||||
- Объект, у которого обнаружен `missing`, невосстановим без бэкапа. Reconciler только помечает его, и API отдаёт 404 на content.
|
||||
- Reconciler хранит все ключи в памяти (ограничение указано выше). `size_mismatch` только сообщается, автоматически не исправляется.
|
||||
- Цель failover-теста выбирается по `X-Instance-Id` обслужившего ответа, а HAProxy раздаёт запросы по кругу, поэтому без вмешательства тест детерминированно останавливал api-2. Для проверки api-1 перед запуском был сделан один лишний запрос через LB (`curl http://localhost:8080/health/live`), сдвигающий очередь; так получен прогон с остановкой api-1.
|
||||
- CI-job со сборкой стенда и `make test` не добавлен: workflow skeleton проверяет только контракт.
|
||||
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
name: mtusi-lab4
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_STORE: ${SESSION_STORE:-valkey}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
|
||||
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT: ${S3_TIMEOUT:-15s}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8}
|
||||
DECODE_BUDGET_BYTES: ${DECODE_BUDGET_BYTES:-134217728}
|
||||
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s}
|
||||
GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.2-alpine3.21
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
|
||||
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||||
secrets: [pg_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
shm_size: 256mb
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"}
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.2-alpine3.21
|
||||
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
|
||||
volumes:
|
||||
- ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro
|
||||
secrets: [valkey_auth]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
# SeaweedFS (master + volume + filer + S3 gateway in one process). Only the S3
|
||||
# port is used, and only inside the data network: nothing is published.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.80
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -s3
|
||||
- -s3.port=8333
|
||||
- -s3.config=/run/secrets/s3_identities
|
||||
- -master.volumeSizeLimitMB=64
|
||||
- -volume.max=100
|
||||
secrets: [s3_identities]
|
||||
volumes:
|
||||
- s3data:/data
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
start_period: 60s
|
||||
retries: 60
|
||||
deploy:
|
||||
resources:
|
||||
# 512M was OOM-killed under 20 parallel 10 MiB PUTs (evidence/memory.json)
|
||||
limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"}
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
# Idempotent bucket bootstrap with the admin identity; the API never sees it.
|
||||
s3-init:
|
||||
image: mtusi-messenger:lab4
|
||||
pull_policy: never
|
||||
command: ["s3-init"]
|
||||
healthcheck:
|
||||
disable: true
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
secrets: [s3_admin_access_key_id, s3_admin_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 64M}
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_BUILD: ${APP_BUILD:-dev}
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
image: mtusi-messenger:lab4
|
||||
pull_policy: never
|
||||
environment: *api-env
|
||||
# No writable filesystem: originals cannot end up on the replica's disk.
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:size=1m
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [app, data]
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
start_period: 5s
|
||||
retries: 3
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"}
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
# Periodic DB<->S3 reconciliation (orphan objects, stale pending uploads,
|
||||
# missing objects). Never touches anything younger than RECONCILE_GRACE.
|
||||
reconciler:
|
||||
image: mtusi-messenger:lab4
|
||||
pull_policy: never
|
||||
command: ["reconcile", "--apply", "--every", "${RECONCILE_EVERY:-10m}"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_MAX_CONNS: "2"
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
RECONCILE_GRACE: ${RECONCILE_GRACE:-15m}
|
||||
healthcheck:
|
||||
disable: true
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
- "[::1]:${APP_PORT:-8080}:8080"
|
||||
networks: [app]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 15
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
app: {}
|
||||
data: {}
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
|
||||
secrets:
|
||||
pg_password:
|
||||
file: ./secrets/pg_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
valkey_auth:
|
||||
file: ./secrets/valkey_auth.conf
|
||||
s3_identities:
|
||||
file: ./secrets/s3_identities.json
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"command": "bootstrap",
|
||||
"date_utc": "2026-10-29T10:38:50Z",
|
||||
"commit": "ec540a1",
|
||||
"result": [
|
||||
{
|
||||
"service": "migrate",
|
||||
"last_log": {
|
||||
"time": "2026-10-29T10:38:51.949471511Z",
|
||||
"level": "INFO",
|
||||
"msg": "migrations up to date",
|
||||
"total": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "s3-init",
|
||||
"last_log": {
|
||||
"time": "2026-10-29T10:38:53.253676835Z",
|
||||
"level": "INFO",
|
||||
"msg": "bucket ready",
|
||||
"bucket": "messenger-attachments",
|
||||
"created": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"last_log": {
|
||||
"time": "2026-10-29T10:38:54.658943295Z",
|
||||
"level": "INFO",
|
||||
"msg": "migrations up to date",
|
||||
"total": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"service": "s3-init",
|
||||
"last_log": {
|
||||
"time": "2026-10-29T10:38:56.030781559Z",
|
||||
"level": "INFO",
|
||||
"msg": "bucket ready",
|
||||
"bucket": "messenger-attachments",
|
||||
"created": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
## make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
## smoke
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"command": "failover-upload",
|
||||
"date_utc": "2026-10-29T10:38:32Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"uploaded_via": "0b30fcb99d56",
|
||||
"stopped_container": "mtusi-lab4-api-2",
|
||||
"reads_after_stop": [
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
},
|
||||
{
|
||||
"instance": "02482efeb4f6",
|
||||
"sha256_ok": true
|
||||
}
|
||||
],
|
||||
"seconds_observed": 9.6,
|
||||
"ps_before": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 running\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running",
|
||||
"ps_during": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 exited\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.095}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
mtusi-lab4-api-1 running
|
||||
mtusi-lab4-api-2 exited
|
||||
|
||||
## other replica (api-1 stopped)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.148}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
mtusi-lab4-api-1 exited
|
||||
mtusi-lab4-api-2 running
|
||||
@@ -0,0 +1,190 @@
|
||||
{
|
||||
"command": "memory",
|
||||
"date_utc": "2026-10-29T10:20:43Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"api_limit": "512M x 2 replicas, 1 CPU each",
|
||||
"gomemlimit": "400MiB",
|
||||
"upload_concurrency_per_replica": 8,
|
||||
"decode_budget_mib": 128,
|
||||
"s3_limit": "1073741824",
|
||||
"runs": [
|
||||
{
|
||||
"concurrency": 1,
|
||||
"repeat": 1,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 1,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 0.35,
|
||||
"p50_ms": 337,
|
||||
"max_ms": 337,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 34.4,
|
||||
"mtusi-lab4-api-2": 64.4
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 29.5,
|
||||
"mtusi-lab4-api-2": 54.8
|
||||
},
|
||||
"max_peak_mib": 64.4,
|
||||
"s3_peak_mib_since_start": 267.1,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 1,
|
||||
"repeat": 2,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 1,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 0.36,
|
||||
"p50_ms": 344,
|
||||
"max_ms": 344,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 64.5,
|
||||
"mtusi-lab4-api-2": 29.9
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 54.5,
|
||||
"mtusi-lab4-api-2": 29.1
|
||||
},
|
||||
"max_peak_mib": 64.5,
|
||||
"s3_peak_mib_since_start": 312.4,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 10,
|
||||
"repeat": 1,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 10,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 4.1,
|
||||
"p50_ms": 3830,
|
||||
"max_ms": 3945,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 148.2,
|
||||
"mtusi-lab4-api-2": 144.4
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 54.8,
|
||||
"mtusi-lab4-api-2": 30.1
|
||||
},
|
||||
"max_peak_mib": 148.2,
|
||||
"s3_peak_mib_since_start": 960.0,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 10,
|
||||
"repeat": 2,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 10,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 3.05,
|
||||
"p50_ms": 2844,
|
||||
"max_ms": 2882,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 143.0,
|
||||
"mtusi-lab4-api-2": 151.7
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 30.3,
|
||||
"mtusi-lab4-api-2": 53.4
|
||||
},
|
||||
"max_peak_mib": 151.7,
|
||||
"s3_peak_mib_since_start": 1025.2,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 20,
|
||||
"repeat": 1,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 20,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 6.35,
|
||||
"p50_ms": 5476,
|
||||
"max_ms": 6062,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 286.0,
|
||||
"mtusi-lab4-api-2": 236.4
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 54.8,
|
||||
"mtusi-lab4-api-2": 29.5
|
||||
},
|
||||
"max_peak_mib": 286.0,
|
||||
"s3_peak_mib_since_start": 1025.2,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 20,
|
||||
"repeat": 2,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 20,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 7.12,
|
||||
"p50_ms": 6097,
|
||||
"max_ms": 6830,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 245.4,
|
||||
"mtusi-lab4-api-2": 263.7
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 30.2,
|
||||
"mtusi-lab4-api-2": 55.1
|
||||
},
|
||||
"max_peak_mib": 263.7,
|
||||
"s3_peak_mib_since_start": 1025.2,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 40,
|
||||
"repeat": 1,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 40,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 12.64,
|
||||
"p50_ms": 9590,
|
||||
"max_ms": 12044,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 264.5,
|
||||
"mtusi-lab4-api-2": 248.3
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 53.9,
|
||||
"mtusi-lab4-api-2": 29.9
|
||||
},
|
||||
"max_peak_mib": 264.5,
|
||||
"s3_peak_mib_since_start": 1030.8,
|
||||
"s3_restarted_during_run": false
|
||||
},
|
||||
{
|
||||
"concurrency": 40,
|
||||
"repeat": 2,
|
||||
"file_bytes": 10420335,
|
||||
"ok": 40,
|
||||
"status_503": 0,
|
||||
"other": [],
|
||||
"wall_s": 11.79,
|
||||
"p50_ms": 8542,
|
||||
"max_ms": 11156,
|
||||
"peak_mib_per_replica": {
|
||||
"mtusi-lab4-api-1": 260.5,
|
||||
"mtusi-lab4-api-2": 311.1
|
||||
},
|
||||
"baseline_current_mib": {
|
||||
"mtusi-lab4-api-1": 29.5,
|
||||
"mtusi-lab4-api-2": 54.5
|
||||
},
|
||||
"max_peak_mib": 311.1,
|
||||
"s3_peak_mib_since_start": 1030.8,
|
||||
"s3_restarted_during_run": false
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"command": "no-local-file",
|
||||
"date_utc": "2026-10-29T10:38:48Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"mtusi-lab4-api-1": {
|
||||
"readonly_rootfs": true,
|
||||
"write_probe": "touch: /app/probe: Read-only file system",
|
||||
"mounts": [
|
||||
"/run/secrets/s3_access_key_id",
|
||||
"/run/secrets/s3_secret_access_key",
|
||||
"/run/secrets/database_url",
|
||||
"/run/secrets/cache_url"
|
||||
],
|
||||
"files_written_since_start": []
|
||||
},
|
||||
"mtusi-lab4-api-2": {
|
||||
"readonly_rootfs": true,
|
||||
"write_probe": "touch: /app/probe: Read-only file system",
|
||||
"mounts": [
|
||||
"/run/secrets/s3_access_key_id",
|
||||
"/run/secrets/s3_secret_access_key",
|
||||
"/run/secrets/database_url",
|
||||
"/run/secrets/cache_url"
|
||||
],
|
||||
"files_written_since_start": []
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
api running Up About a minute (healthy)
|
||||
api running Up About a minute (healthy)
|
||||
lb running Up 11 minutes (healthy)
|
||||
migrate exited Exited (0) About a minute ago
|
||||
postgres running Up 11 minutes (healthy)
|
||||
reconciler running Up About a minute
|
||||
s3 running Up 38 seconds (healthy)
|
||||
s3-init exited Exited (0) About a minute ago
|
||||
valkey running Up 11 minutes (healthy)
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
api running Up 7 seconds (healthy)
|
||||
api running Up 7 seconds (healthy)
|
||||
lb running Up 1 second (health: starting)
|
||||
migrate exited Exited (0) 8 seconds ago
|
||||
postgres running Up 46 seconds (healthy)
|
||||
reconciler running Up 7 seconds
|
||||
s3 running Up 36 seconds (healthy)
|
||||
s3-init exited Exited (0) 8 seconds ago
|
||||
valkey running Up 46 seconds (healthy)
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"command": "private",
|
||||
"date_utc": "2026-10-29T10:38:17Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"object_key_shape": "attachments/<chat_id>/<attachment_id> (server generated)",
|
||||
"key_contains_filename": false,
|
||||
"anonymous_get": 403,
|
||||
"anonymous_list": "denied (HTTP 403)",
|
||||
"api_identity_get": 200,
|
||||
"api_identity_get_sha256_matches": true,
|
||||
"api_identity_create_bucket": 403,
|
||||
"anonymous_create_bucket": 403,
|
||||
"published_s3_ports": "invalid IP:0",
|
||||
"outsider_via_api_metadata": 404,
|
||||
"outsider_via_api_content": 404,
|
||||
"api_identity_put_other_bucket": "{\"cmd\": \"put\", \"as\": \"api\", \"status\": 403}"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"command": "s3-outage",
|
||||
"date_utc": "2026-10-29T11:45:59Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"upload_status": 503,
|
||||
"upload_ms": 2048,
|
||||
"download_status": 503,
|
||||
"download_ms": 2018,
|
||||
"metadata_status": 200,
|
||||
"text_message_status": 201,
|
||||
"ready_status": 200,
|
||||
"pending_rows_left": 0,
|
||||
"upload_recovered_after_s3_healthy_s": 0.1,
|
||||
"old_file_intact": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
[
|
||||
{
|
||||
"test": "one session served by >=2 replicas without re-login (no stickiness)",
|
||||
"result": "PASS",
|
||||
"detail": "distribution [15, 15]"
|
||||
},
|
||||
{
|
||||
"test": "X-Instance-Id is present on errors and health too",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "Idempotency-Key missing or malformed -> 400, nothing stored",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "concurrent replay of one key through different replicas -> exactly one message",
|
||||
"result": "PASS",
|
||||
"detail": "24 parallel requests via 2 replicas -> 1 message"
|
||||
},
|
||||
{
|
||||
"test": "replay is semantic: JSON key order/whitespace does not matter; other body -> 409",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "key scope = user + chat: same key is independent for other user / other chat",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "logout is visible on every replica immediately (shared sessions)",
|
||||
"result": "PASS",
|
||||
"detail": "revoked token rejected by 2 replicas"
|
||||
},
|
||||
{
|
||||
"test": "replay re-checks the session and chat access",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "PNG and JPEG upload/download byte-exact with true Content-Type",
|
||||
"result": "PASS",
|
||||
"detail": "png 111817 B, jpeg 2565 B"
|
||||
},
|
||||
{
|
||||
"test": "size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413",
|
||||
"result": "PASS",
|
||||
"detail": "accepted 10420335 B, rejected 10485761 B"
|
||||
},
|
||||
{
|
||||
"test": "decompression bomb (400M px header, tiny body) -> 413 before decoding",
|
||||
"result": "PASS",
|
||||
"detail": "83 B rejected in 25 ms"
|
||||
},
|
||||
{
|
||||
"test": "corrupted images and MIME spoofing -> 415, never 500",
|
||||
"result": "PASS",
|
||||
"detail": "9 spoof/corrupt variants -> 415"
|
||||
},
|
||||
{
|
||||
"test": "dangerous filenames -> 400; Unicode display name kept as-is",
|
||||
"result": "PASS",
|
||||
"detail": "12 bad names rejected"
|
||||
},
|
||||
{
|
||||
"test": "attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "idempotency: missing attachment_ids == []; other attachment list -> 409",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back",
|
||||
"result": "PASS",
|
||||
"detail": "10 uploads via 2 replicas"
|
||||
},
|
||||
{
|
||||
"test": "ACL: outsider gets 404 on upload, metadata, content; no token -> 401",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"test": "message attachment_ids: other chat / foreign / unknown -> 404, nothing stored",
|
||||
"result": "PASS",
|
||||
"detail": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
PASS one session served by >=2 replicas without re-login (no stickiness) - distribution [15, 15]
|
||||
PASS X-Instance-Id is present on errors and health too -
|
||||
PASS Idempotency-Key missing or malformed -> 400, nothing stored -
|
||||
PASS concurrent replay of one key through different replicas -> exactly one message - 24 parallel requests via 2 replicas -> 1 message
|
||||
PASS replay is semantic: JSON key order/whitespace does not matter; other body -> 409 -
|
||||
PASS key scope = user + chat: same key is independent for other user / other chat -
|
||||
PASS logout is visible on every replica immediately (shared sessions) - revoked token rejected by 2 replicas
|
||||
PASS replay re-checks the session and chat access -
|
||||
PASS PNG and JPEG upload/download byte-exact with true Content-Type - png 111817 B, jpeg 2565 B
|
||||
PASS size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413 - accepted 10420335 B, rejected 10485761 B
|
||||
PASS decompression bomb (400M px header, tiny body) -> 413 before decoding - 83 B rejected in 25 ms
|
||||
PASS corrupted images and MIME spoofing -> 415, never 500 - 9 spoof/corrupt variants -> 415
|
||||
PASS dangerous filenames -> 400; Unicode display name kept as-is - 12 bad names rejected
|
||||
PASS attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400 -
|
||||
PASS idempotency: missing attachment_ids == []; other attachment list -> 409 -
|
||||
PASS 10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back - 10 uploads via 2 replicas
|
||||
PASS ACL: outsider gets 404 on upload, metadata, content; no token -> 401 -
|
||||
PASS message attachment_ids: other chat / foreign / unknown -> 404, nothing stored -
|
||||
18/18 passed
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"command": "cost",
|
||||
"date_utc": "2026-10-29T10:33:43Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"object_size_bytes": 100,
|
||||
"reconciler_limits": "0.5 CPU, 128M",
|
||||
"s3": "SeaweedFS 3.80, 1 CPU, 1G",
|
||||
"runs": [
|
||||
{
|
||||
"bench_objects": 1000,
|
||||
"repeat": 1,
|
||||
"objects_scanned": 1229,
|
||||
"rows_scanned": 1232,
|
||||
"list_ms": 52,
|
||||
"db_ms": 9,
|
||||
"total_ms": 61,
|
||||
"per_object_us": 50.0,
|
||||
"container_memory_peak_mib": 10.7,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 1000,
|
||||
"repeat": 2,
|
||||
"objects_scanned": 1229,
|
||||
"rows_scanned": 1232,
|
||||
"list_ms": 52,
|
||||
"db_ms": 6,
|
||||
"total_ms": 59,
|
||||
"per_object_us": 48.1,
|
||||
"container_memory_peak_mib": 10.1,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 1000,
|
||||
"repeat": 3,
|
||||
"objects_scanned": 1229,
|
||||
"rows_scanned": 1232,
|
||||
"list_ms": 52,
|
||||
"db_ms": 7,
|
||||
"total_ms": 60,
|
||||
"per_object_us": 48.8,
|
||||
"container_memory_peak_mib": 10.9,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 5000,
|
||||
"repeat": 1,
|
||||
"objects_scanned": 5229,
|
||||
"rows_scanned": 5232,
|
||||
"list_ms": 258,
|
||||
"db_ms": 67,
|
||||
"total_ms": 326,
|
||||
"per_object_us": 62.5,
|
||||
"container_memory_peak_mib": 17.2,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 5000,
|
||||
"repeat": 2,
|
||||
"objects_scanned": 5229,
|
||||
"rows_scanned": 5232,
|
||||
"list_ms": 345,
|
||||
"db_ms": 62,
|
||||
"total_ms": 408,
|
||||
"per_object_us": 78.2,
|
||||
"container_memory_peak_mib": 16.9,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 5000,
|
||||
"repeat": 3,
|
||||
"objects_scanned": 5229,
|
||||
"rows_scanned": 5232,
|
||||
"list_ms": 263,
|
||||
"db_ms": 70,
|
||||
"total_ms": 335,
|
||||
"per_object_us": 64.1,
|
||||
"container_memory_peak_mib": 15.8,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 20000,
|
||||
"repeat": 1,
|
||||
"objects_scanned": 20229,
|
||||
"rows_scanned": 20232,
|
||||
"list_ms": 1425,
|
||||
"db_ms": 95,
|
||||
"total_ms": 1523,
|
||||
"per_object_us": 75.3,
|
||||
"container_memory_peak_mib": 30.8,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 20000,
|
||||
"repeat": 2,
|
||||
"objects_scanned": 20229,
|
||||
"rows_scanned": 20232,
|
||||
"list_ms": 1537,
|
||||
"db_ms": 41,
|
||||
"total_ms": 1618,
|
||||
"per_object_us": 80.0,
|
||||
"container_memory_peak_mib": 31.1,
|
||||
"findings": 0
|
||||
},
|
||||
{
|
||||
"bench_objects": 20000,
|
||||
"repeat": 3,
|
||||
"objects_scanned": 20229,
|
||||
"rows_scanned": 20232,
|
||||
"list_ms": 1321,
|
||||
"db_ms": 76,
|
||||
"total_ms": 1404,
|
||||
"per_object_us": 69.4,
|
||||
"container_memory_peak_mib": 31.2,
|
||||
"findings": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"command": "faults",
|
||||
"date_utc": "2026-10-29T10:31:36Z",
|
||||
"commit": "ec540a1",
|
||||
"result": {
|
||||
"grace_used": "1m",
|
||||
"faults": {
|
||||
"F1_orphan_object_no_row": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
||||
"F2_metadata_row_lost(U3)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
||||
"F3_object_lost_of_used(U2)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
||||
"F4_crash_after_put_before_ready": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
||||
"F5_crash_before_put": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
||||
"F6_F7_fresh_upload_in_progress": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/693c91e3-43f6-46f6-9bef-2b0de987546a",
|
||||
"F8_fresh_orphan_object": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc"
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"step": "dry-run",
|
||||
"report": {
|
||||
"mode": "dry-run",
|
||||
"objects_scanned": 233,
|
||||
"rows_scanned": 235,
|
||||
"orphan_objects": 3,
|
||||
"fresh_skipped": 1,
|
||||
"stale_pending": 3,
|
||||
"missing_objects": 1,
|
||||
"size_mismatch": 0,
|
||||
"recovered": 0,
|
||||
"fixed": 0,
|
||||
"errors": 0,
|
||||
"total_ms": 34
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"kind": "stale_pending",
|
||||
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
|
||||
"attachment_id": "34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
|
||||
"action": "would delete row and object"
|
||||
},
|
||||
{
|
||||
"kind": "stale_pending",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
||||
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
|
||||
"action": "would delete row and object"
|
||||
},
|
||||
{
|
||||
"kind": "stale_pending",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
||||
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
||||
"action": "would delete row and object"
|
||||
},
|
||||
{
|
||||
"kind": "missing_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
||||
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
|
||||
"action": "would mark missing (metadata kept)"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/5585fa4c-c149-49f6-b5d1-1196038eba43",
|
||||
"action": "would delete object"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
|
||||
"action": "skip: younger than grace"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
||||
"action": "would delete object"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
||||
"action": "would delete object"
|
||||
}
|
||||
],
|
||||
"state_before": {
|
||||
"s3_objects": 233,
|
||||
"db_rows": 235,
|
||||
"db_pending": 4,
|
||||
"db_missing_marked": 4
|
||||
},
|
||||
"state_after": {
|
||||
"s3_objects": 233,
|
||||
"db_rows": 235,
|
||||
"db_pending": 4,
|
||||
"db_missing_marked": 4
|
||||
},
|
||||
"dry_run_changed_nothing": true,
|
||||
"this_experiment": {
|
||||
"stale_pending": 2,
|
||||
"missing_object": 1,
|
||||
"orphan_object/skip": 1,
|
||||
"orphan_object": 2
|
||||
},
|
||||
"other_findings_outside_experiment": {
|
||||
"stale_pending": 1,
|
||||
"orphan_object": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "apply #1",
|
||||
"report": {
|
||||
"mode": "apply",
|
||||
"objects_scanned": 233,
|
||||
"rows_scanned": 235,
|
||||
"orphan_objects": 3,
|
||||
"fresh_skipped": 1,
|
||||
"stale_pending": 3,
|
||||
"missing_objects": 1,
|
||||
"size_mismatch": 0,
|
||||
"recovered": 0,
|
||||
"fixed": 7,
|
||||
"errors": 0,
|
||||
"total_ms": 46
|
||||
},
|
||||
"this_experiment": {
|
||||
"stale_pending": 2,
|
||||
"missing_object": 1,
|
||||
"orphan_object/skip": 1,
|
||||
"orphan_object": 2
|
||||
},
|
||||
"items": [
|
||||
{
|
||||
"kind": "stale_pending",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
|
||||
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
|
||||
"action": "deleted row and object"
|
||||
},
|
||||
{
|
||||
"kind": "stale_pending",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
||||
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
|
||||
"action": "deleted row and object"
|
||||
},
|
||||
{
|
||||
"kind": "missing_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
|
||||
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
|
||||
"action": "marked missing (metadata kept)"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
|
||||
"action": "skip: younger than grace"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
|
||||
"action": "deleted object"
|
||||
},
|
||||
{
|
||||
"kind": "orphan_object",
|
||||
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
|
||||
"action": "deleted object"
|
||||
}
|
||||
],
|
||||
"state_after": {
|
||||
"s3_objects": 228,
|
||||
"db_rows": 232,
|
||||
"db_pending": 1,
|
||||
"db_missing_marked": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "apply #2 (repeat)",
|
||||
"report": {
|
||||
"mode": "apply",
|
||||
"objects_scanned": 228,
|
||||
"rows_scanned": 232,
|
||||
"orphan_objects": 0,
|
||||
"fresh_skipped": 1,
|
||||
"stale_pending": 0,
|
||||
"missing_objects": 0,
|
||||
"size_mismatch": 0,
|
||||
"recovered": 0,
|
||||
"fixed": 0,
|
||||
"errors": 0,
|
||||
"total_ms": 25
|
||||
},
|
||||
"this_experiment": {
|
||||
"orphan_object/skip": 1
|
||||
},
|
||||
"state_after": {
|
||||
"s3_objects": 228,
|
||||
"db_rows": 232,
|
||||
"db_pending": 1,
|
||||
"db_missing_marked": 5
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "apply after restoring U2 object",
|
||||
"report": {
|
||||
"mode": "apply",
|
||||
"objects_scanned": 229,
|
||||
"rows_scanned": 232,
|
||||
"orphan_objects": 0,
|
||||
"fresh_skipped": 1,
|
||||
"stale_pending": 0,
|
||||
"missing_objects": 0,
|
||||
"size_mismatch": 0,
|
||||
"recovered": 1,
|
||||
"fixed": 1,
|
||||
"errors": 0,
|
||||
"total_ms": 27
|
||||
}
|
||||
},
|
||||
{
|
||||
"step": "dry-run with production grace 15m",
|
||||
"report": {
|
||||
"mode": "dry-run",
|
||||
"objects_scanned": 229,
|
||||
"rows_scanned": 232,
|
||||
"orphan_objects": 0,
|
||||
"fresh_skipped": 1,
|
||||
"stale_pending": 0,
|
||||
"missing_objects": 0,
|
||||
"size_mismatch": 0,
|
||||
"recovered": 0,
|
||||
"fixed": 0,
|
||||
"errors": 0,
|
||||
"total_ms": 31
|
||||
}
|
||||
}
|
||||
],
|
||||
"verify": {
|
||||
"U1_used_healthy_download_sha_ok": true,
|
||||
"U2_metadata_kept": 200,
|
||||
"U2_content_status": 404,
|
||||
"U2_marked_missing": true,
|
||||
"message_still_references_U1_U2": true,
|
||||
"F1_orphan_deleted": true,
|
||||
"F2_orphan_of_lost_row_deleted": true,
|
||||
"F4_row_and_object_deleted": true,
|
||||
"F5_row_deleted": true,
|
||||
"F7_fresh_pending_row_kept": true,
|
||||
"F7_fresh_object_kept": true,
|
||||
"F8_fresh_orphan_kept": true,
|
||||
"U2_restored_and_unmarked": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold obsolete 5s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s
|
||||
timeout client 75s
|
||||
timeout server 75s
|
||||
timeout http-request 5s
|
||||
timeout http-keep-alive 10s
|
||||
timeout check 500ms
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4
|
||||
server-template api 8 api:8080
|
||||
|
||||
frontend internal_stats
|
||||
bind :8404
|
||||
monitor-uri /healthz
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 5s
|
||||
@@ -0,0 +1,8 @@
|
||||
bind 0.0.0.0
|
||||
port 6379
|
||||
protected-mode yes
|
||||
save ""
|
||||
appendonly no
|
||||
maxmemory 64mb
|
||||
maxmemory-policy volatile-ttl
|
||||
include /run/secrets/valkey_auth
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import secrets
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, value):
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text().strip()
|
||||
path.write_bytes((value + '\n').encode('utf-8'))
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
pg = write('pg_password', secrets.token_urlsafe(24))
|
||||
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
|
||||
vk_conf = SECRETS / 'valkey_auth.conf'
|
||||
if vk_conf.exists():
|
||||
vk = vk_conf.read_text().split()[-1]
|
||||
print('keep secrets/valkey_auth.conf')
|
||||
else:
|
||||
vk = secrets.token_urlsafe(24)
|
||||
write('valkey_auth.conf', 'requirepass ' + vk)
|
||||
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
|
||||
# Two S3 identities: admin (bucket bootstrap only) and api (read/write/list
|
||||
# of the one bucket). SeaweedFS has no anonymous identity -> bucket is private.
|
||||
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
|
||||
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
|
||||
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
|
||||
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
|
||||
bucket = 'messenger-attachments'
|
||||
identities = {'identities': [
|
||||
{'name': 'bootstrap-admin',
|
||||
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
|
||||
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
|
||||
{'name': 'messenger-api',
|
||||
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
|
||||
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
|
||||
]}
|
||||
write('s3_identities.json', json.dumps(identities, indent=2))
|
||||
env = ROOT / '.env'
|
||||
if not env.exists():
|
||||
shutil.copy(ROOT / '.env.example', env)
|
||||
print('created .env from .env.example')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,269 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
|
||||
|
||||
python scripts/lab4ops.py private # bucket privacy outside the API
|
||||
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
|
||||
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
|
||||
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
|
||||
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
|
||||
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
|
||||
|
||||
Each command prints JSON and, with --output, stores it in evidence/.
|
||||
Synthetic data only; tokens/credentials are never printed.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
import imagegen # noqa: E402
|
||||
from lablib import api_containers, sh, wait_healthy # noqa: E402
|
||||
|
||||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
NETWORK = 'mtusi-lab4_data'
|
||||
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
|
||||
|
||||
|
||||
def winpath(p):
|
||||
s = str(p)
|
||||
return s.replace('\\', '/')
|
||||
|
||||
|
||||
def s3tool(*args, stdin=None):
|
||||
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
|
||||
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
|
||||
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
|
||||
return json.loads(r.stdout.strip().splitlines()[-1])
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
|
||||
|
||||
|
||||
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
|
||||
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment' if expected == 201 else None)
|
||||
|
||||
|
||||
def cmd_private(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)['json']
|
||||
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
|
||||
res = {
|
||||
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
|
||||
'key_contains_filename': 'sample' in key,
|
||||
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
|
||||
'anonymous_list': None,
|
||||
'api_identity_get': s3tool('get', key)['status'],
|
||||
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
|
||||
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
|
||||
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
|
||||
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
|
||||
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
|
||||
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
|
||||
}
|
||||
try:
|
||||
s3tool('list', 'attachments/', '--as', 'anonymous')
|
||||
res['anonymous_list'] = 200
|
||||
except RuntimeError as exc:
|
||||
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
|
||||
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
|
||||
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
|
||||
res['api_identity_put_other_bucket'] = subprocess.run(
|
||||
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
|
||||
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
|
||||
'api identity has more rights than expected')
|
||||
return res
|
||||
|
||||
|
||||
def cmd_failover_upload(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
r = upload(c, alice, chat)
|
||||
att, served_by = r['json'], r['headers']['X-Instance-Id']
|
||||
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
|
||||
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
|
||||
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
||||
sh('docker', 'stop', '-t', '15', target[0]['id'])
|
||||
t0 = time.monotonic()
|
||||
reads = []
|
||||
try:
|
||||
for _ in range(10):
|
||||
try:
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||||
reads.append({'instance': d['headers']['X-Instance-Id'],
|
||||
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
|
||||
except Failure as exc:
|
||||
reads.append({'error': str(exc)[-60:]})
|
||||
time.sleep(0.3)
|
||||
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
|
||||
finally:
|
||||
sh('docker', 'start', target[0]['id'])
|
||||
wait_healthy(target[0]['id'])
|
||||
ok = [x for x in reads if x.get('sha256_ok')]
|
||||
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
|
||||
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
|
||||
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
|
||||
|
||||
|
||||
def cmd_s3_outage(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)['json']
|
||||
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
|
||||
sh('docker', 'compose', 'stop', 's3')
|
||||
res = {}
|
||||
try:
|
||||
u = upload(c, alice, chat, expected=503)
|
||||
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
|
||||
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
|
||||
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
|
||||
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
|
||||
res['ready_status'] = c.call('GET', '/health/ready')['status']
|
||||
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
|
||||
res['pending_rows_left'] = int(psql(
|
||||
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
|
||||
finally:
|
||||
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
|
||||
t0 = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
upload(c, alice, chat)
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
|
||||
time.sleep(0.5)
|
||||
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
|
||||
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||||
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
|
||||
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
|
||||
'S3 outage is not isolated to attachment endpoints')
|
||||
return res
|
||||
|
||||
|
||||
def cmd_no_local_file(a):
|
||||
out = {}
|
||||
for r in api_containers():
|
||||
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
|
||||
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
|
||||
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
|
||||
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
|
||||
files = sh('docker', 'exec', r['id'], 'sh', '-c',
|
||||
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
|
||||
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
|
||||
'! -name .dockerenv 2>/dev/null | head -20')
|
||||
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
|
||||
'mounts': [m.get('Destination') for m in mounts],
|
||||
'files_written_since_start': files.splitlines()}
|
||||
check(ro == 'true' and not files.strip(), 'API container can store files locally')
|
||||
return out
|
||||
|
||||
|
||||
def cmd_bootstrap(a):
|
||||
runs = []
|
||||
for _ in range(2):
|
||||
for svc in ('migrate', 's3-init'):
|
||||
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
|
||||
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
|
||||
return runs
|
||||
|
||||
|
||||
def peak(cid):
|
||||
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
|
||||
return int(v[0]) / 2**20, int(v[1]) / 2**20
|
||||
|
||||
|
||||
def cmd_memory(a):
|
||||
big = imagegen.near_limit_png()
|
||||
c = Client(a.base_url, timeout=120)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
results = []
|
||||
for conc in a.levels:
|
||||
for rep in range(a.repeat):
|
||||
sh('docker', 'compose', 'restart', 'api')
|
||||
for r in api_containers():
|
||||
check(wait_healthy(r['id'], 60), 'api not healthy')
|
||||
time.sleep(1)
|
||||
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
|
||||
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
|
||||
upload(c, alice, chat, big) # warm-up allocations of one replica
|
||||
base = {r['name']: peak(r['id']) for r in api_containers()}
|
||||
barrier = threading.Barrier(conc)
|
||||
|
||||
def one(i):
|
||||
local = Client(a.base_url, timeout=120)
|
||||
barrier.wait()
|
||||
try:
|
||||
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
|
||||
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
|
||||
return x['status'], x['elapsed']
|
||||
except Failure as exc:
|
||||
return str(exc)[-40:], 0
|
||||
|
||||
t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(conc) as pool:
|
||||
res = list(pool.map(one, range(conc)))
|
||||
wall = time.monotonic() - t0
|
||||
after = {r['name']: peak(r['id']) for r in api_containers()}
|
||||
lat = sorted(e for s, e in res if s == 201)
|
||||
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
|
||||
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
|
||||
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
|
||||
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
|
||||
'max_ms': round(1000 * lat[-1]) if lat else None,
|
||||
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
|
||||
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
|
||||
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
|
||||
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
|
||||
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
|
||||
print(json.dumps(row, ensure_ascii=False), flush=True)
|
||||
results.append(row)
|
||||
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
|
||||
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
|
||||
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
|
||||
'runs': results}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--output')
|
||||
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
|
||||
p.add_argument('--repeat', type=int, default=2)
|
||||
a = p.parse_args()
|
||||
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
|
||||
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
|
||||
text = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if a.output:
|
||||
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
||||
print('PASS: ' + a.cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,130 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
PROJECT = 'mtusi-lab4'
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
|
||||
if check and r.returncode != 0:
|
||||
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def api_containers(all_states=False):
|
||||
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
|
||||
'--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api',
|
||||
'--format', '{{json .}}')
|
||||
rows = []
|
||||
for line in out.splitlines():
|
||||
d = json.loads(line)
|
||||
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
|
||||
return sorted(rows, key=lambda r: r['name'])
|
||||
|
||||
|
||||
def health(cid):
|
||||
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
|
||||
|
||||
|
||||
def wait_healthy(cid, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if health(cid) == 'healthy':
|
||||
return True
|
||||
time.sleep(0.25)
|
||||
return False
|
||||
|
||||
|
||||
def ps_snapshot():
|
||||
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
|
||||
|
||||
|
||||
class WriteLoad:
|
||||
|
||||
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
|
||||
self.base = base_url
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.pause = pause
|
||||
self.events = []
|
||||
self.confirmed = []
|
||||
self.retries = 0
|
||||
self._lock = threading.Lock()
|
||||
self._stop = threading.Event()
|
||||
self._threads = []
|
||||
c = Client(base_url, timeout=10)
|
||||
self.alice, self.bob, _eve, self.chat = c.bootstrap()
|
||||
self.reader = c
|
||||
|
||||
def _worker(self, wid):
|
||||
c = Client(self.base, timeout=self.timeout)
|
||||
key = text = None
|
||||
n = 0
|
||||
while not self._stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load-%d-%d-%s' % (wid, n, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = c.message(self.alice, self.chat, text, key=key)
|
||||
with self._lock:
|
||||
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
|
||||
self.confirmed.append(r['json'])
|
||||
key = None
|
||||
n += 1
|
||||
except Failure as exc:
|
||||
with self._lock:
|
||||
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
|
||||
self.retries += 1
|
||||
time.sleep(0.05)
|
||||
if self.pause:
|
||||
time.sleep(self.pause)
|
||||
|
||||
def start(self):
|
||||
for i in range(self.workers):
|
||||
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
|
||||
t.start()
|
||||
self._threads.append(t)
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
for t in self._threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
def window(self, a, b):
|
||||
ev = [e for e in self.events if a <= e[0] < b]
|
||||
ok = [e for e in ev if e[1]]
|
||||
lat = sorted(e[3] for e in ok)
|
||||
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
|
||||
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
|
||||
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
|
||||
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
|
||||
|
||||
def verify_history(self):
|
||||
items, cursor = [], None
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
|
||||
while True:
|
||||
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
|
||||
actor=self.bob, schema='MessagePage')['json']
|
||||
items += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if not cursor:
|
||||
break
|
||||
by_id = {m['id']: m for m in items}
|
||||
texts = [m['text'] for m in items]
|
||||
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
|
||||
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
|
||||
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
|
||||
'duplicate_texts': len(texts) - len(set(texts)),
|
||||
'unique_confirmed': len({m['id'] for m in self.confirmed})}
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop lb api reconciler || exit /b 1
|
||||
docker compose restart postgres valkey s3 || exit /b 1
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3 || exit /b 1
|
||||
docker compose start api reconciler lb || exit /b 1
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Restart API and ALL state stores (PostgreSQL, Valkey, S3) keeping volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop lb api reconciler
|
||||
docker compose restart postgres valkey s3
|
||||
# no recreation, only waits until the stores report healthy again
|
||||
docker compose up -d --no-recreate --wait postgres valkey s3
|
||||
docker compose start api reconciler lb
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
|
||||
set CID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api"') do set CID=%%i
|
||||
if "%CID%"=="" exit /b 3
|
||||
docker stop -t 15 %CID% >nul || exit /b 1
|
||||
echo stopped api container %CID%
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab4" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
|
||||
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
|
||||
docker stop -t 15 "$cid" >/dev/null
|
||||
echo "stopped api container $cid"
|
||||
@@ -0,0 +1,254 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
|
||||
|
||||
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
|
||||
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
|
||||
|
||||
The periodic `reconciler` service is stopped for the duration (it would race the
|
||||
experiment) and started again at the end. Synthetic data only.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'scripts'))
|
||||
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
|
||||
from lablib import sh # noqa: E402
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
|
||||
def s3put_bytes(key, data):
|
||||
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
|
||||
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
|
||||
r = subprocess.run(cmd, input=data, capture_output=True)
|
||||
check(r.returncode == 0, 's3 put failed')
|
||||
return json.loads(r.stdout.decode().strip().splitlines()[-1])
|
||||
|
||||
|
||||
def reconcile(*args):
|
||||
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
|
||||
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
|
||||
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
|
||||
'&& echo && cat /sys/fs/cgroup/memory.peak')
|
||||
lines = [x for x in out.splitlines() if x.strip()]
|
||||
rep = json.loads(lines[-2])
|
||||
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
|
||||
return rep
|
||||
|
||||
|
||||
def counts(prefix='attachments/'):
|
||||
return {'s3_objects': s3tool('count', prefix)['count'],
|
||||
'db_rows': int(psql('SELECT count(*) FROM attachments')),
|
||||
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
|
||||
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
|
||||
|
||||
|
||||
def summary(rep):
|
||||
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
|
||||
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
|
||||
return {k: rep[k] for k in keys}
|
||||
|
||||
|
||||
def scoped(rep, prefix, invert=False):
|
||||
out = {}
|
||||
for i in rep['items']:
|
||||
if i.get('key', '').startswith(prefix) == invert:
|
||||
continue
|
||||
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
|
||||
out[k] = out.get(k, 0) + 1
|
||||
return out
|
||||
|
||||
|
||||
def insert_pending(att_id, chat, owner, key, age):
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
|
||||
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
|
||||
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
|
||||
% (att_id, chat, owner, key, age, age))
|
||||
|
||||
|
||||
def exists(key):
|
||||
return s3tool('get', key)['status'] == 200
|
||||
|
||||
|
||||
def cmd_faults(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
|
||||
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
|
||||
|
||||
# healthy, referenced attachment (must survive everything)
|
||||
u1 = upload(c, alice, chat)['json']
|
||||
# referenced attachment whose object will be lost (S3-side fault)
|
||||
u2 = upload(c, alice, chat)['json']
|
||||
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
|
||||
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
|
||||
u3 = upload(c, alice, chat)['json']
|
||||
u3_key = key_of(u3['id'])
|
||||
|
||||
f = {}
|
||||
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
|
||||
s3tool('put', f['F1_orphan_object_no_row'])
|
||||
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
|
||||
f['F2_metadata_row_lost(U3)'] = u3_key
|
||||
s3tool('delete', key_of(u2['id']))
|
||||
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
|
||||
f4_id = str(uuid.uuid4())
|
||||
f['F4_crash_after_put_before_ready'] = prefix + f4_id
|
||||
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
|
||||
s3tool('put', f['F4_crash_after_put_before_ready'])
|
||||
f5_id = str(uuid.uuid4())
|
||||
f['F5_crash_before_put'] = prefix + f5_id
|
||||
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
|
||||
|
||||
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
|
||||
time.sleep(a.wait)
|
||||
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
|
||||
f7_id = str(uuid.uuid4())
|
||||
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
|
||||
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
|
||||
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
|
||||
f8 = prefix + str(uuid.uuid4())
|
||||
f['F8_fresh_orphan_object'] = f8
|
||||
s3tool('put', f8)
|
||||
|
||||
steps = []
|
||||
before = counts()
|
||||
dry = reconcile('--grace', a.grace)
|
||||
after_dry = counts()
|
||||
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
|
||||
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
|
||||
check(before == after_dry, 'dry-run modified state')
|
||||
mine = scoped(dry, prefix)
|
||||
steps[-1]['this_experiment'] = mine
|
||||
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
|
||||
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
|
||||
'detection mismatch: %s' % mine)
|
||||
|
||||
ap = reconcile('--apply', '--grace', a.grace)
|
||||
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
|
||||
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
|
||||
again = reconcile('--apply', '--grace', a.grace)
|
||||
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
|
||||
'state_after': counts()})
|
||||
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
|
||||
|
||||
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
|
||||
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
|
||||
verify = {
|
||||
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
|
||||
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
|
||||
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
|
||||
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
|
||||
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
|
||||
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
|
||||
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
|
||||
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
|
||||
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
|
||||
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
|
||||
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
|
||||
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
|
||||
'F8_fresh_orphan_kept': exists(f8),
|
||||
}
|
||||
for k, v in verify.items():
|
||||
check(v is True or v in (200, 404), k)
|
||||
|
||||
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
|
||||
s3put_bytes(key_of(u2['id']), PNG)
|
||||
rec = reconcile('--apply', '--grace', a.grace)
|
||||
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
|
||||
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
|
||||
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
|
||||
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
|
||||
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
|
||||
check(verify['U2_restored_and_unmarked'], 'restore not detected')
|
||||
|
||||
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
|
||||
prod = reconcile('--grace', '15m')
|
||||
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
|
||||
# clean the fresh leftovers with the regular procedure so the next run starts clean
|
||||
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
|
||||
|
||||
|
||||
def cmd_cost(a):
|
||||
c = Client(a.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
run = uuid.uuid4().hex[:8]
|
||||
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
|
||||
created = 0
|
||||
results = []
|
||||
try:
|
||||
for n in sorted(a.sizes):
|
||||
add = n - created
|
||||
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
|
||||
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
|
||||
" width, height, object_key, status, created_at, updated_at)"
|
||||
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
|
||||
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
|
||||
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
|
||||
t0 = time.monotonic()
|
||||
put = json.loads(subprocess.run(
|
||||
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
||||
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
|
||||
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
|
||||
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
|
||||
created = n
|
||||
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
|
||||
for r in range(a.repeat):
|
||||
rep = reconcile('--grace', '15m')
|
||||
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
|
||||
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
|
||||
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
|
||||
'container_memory_peak_mib': rep['container_memory_peak_mib'],
|
||||
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
|
||||
print(json.dumps(row), flush=True)
|
||||
results.append(row)
|
||||
finally:
|
||||
if created:
|
||||
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
|
||||
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
|
||||
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
|
||||
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
|
||||
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
|
||||
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
|
||||
input='\n'.join(keys), capture_output=True, text=True)
|
||||
if keys_file.exists():
|
||||
keys_file.unlink()
|
||||
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
|
||||
'runs': results}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['faults', 'cost'])
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--grace', default='1m')
|
||||
p.add_argument('--wait', type=int, default=65)
|
||||
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
|
||||
p.add_argument('--repeat', type=int, default=3)
|
||||
p.add_argument('--output')
|
||||
a = p.parse_args()
|
||||
sh('docker', 'compose', 'stop', 'reconciler')
|
||||
try:
|
||||
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
|
||||
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
|
||||
finally:
|
||||
sh('docker', 'compose', 'start', 'reconciler')
|
||||
text = json.dumps(res, ensure_ascii=False, indent=2)
|
||||
if a.output:
|
||||
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(a.output).write_text(text + '\n', encoding='utf-8')
|
||||
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
|
||||
print('PASS: track2 ' + a.cmd)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
|
||||
|
||||
Runs inside a throwaway container on the compose `data` network (S3 is not
|
||||
published to the host), see run_in_net() in lab4ops.py. Credentials are read
|
||||
from mounted secret files and never printed.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from urllib.parse import quote
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
HOST = os.environ.get('S3_HOST', 's3:8333')
|
||||
REGION = os.environ.get('S3_REGION', 'us-east-1')
|
||||
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
|
||||
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
|
||||
|
||||
|
||||
def creds(identity):
|
||||
if identity == 'anonymous':
|
||||
return None
|
||||
prefix = 's3_admin_' if identity == 'admin' else 's3_'
|
||||
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
|
||||
return read('access_key_id'), read('secret_access_key')
|
||||
|
||||
|
||||
def _h(key, msg):
|
||||
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
|
||||
|
||||
|
||||
_local = threading.local()
|
||||
|
||||
|
||||
def request(method, path, identity='api', body=b'', query=None):
|
||||
query = query or {}
|
||||
headers = {'host': HOST}
|
||||
payload_hash = hashlib.sha256(body).hexdigest()
|
||||
c = creds(identity)
|
||||
if c:
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
||||
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
|
||||
signed = ';'.join(sorted(headers))
|
||||
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
|
||||
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
|
||||
scope = '%s/%s/s3/aws4_request' % (date, REGION)
|
||||
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
|
||||
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
|
||||
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
|
||||
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
|
||||
c[0], scope, signed, sig)
|
||||
url = quote(path, safe='/-_.~')
|
||||
if query:
|
||||
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
|
||||
conn = getattr(_local, 'conn', None)
|
||||
if conn is None:
|
||||
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
|
||||
try:
|
||||
conn.request(method, url, body=body or None, headers=headers)
|
||||
r = conn.getresponse()
|
||||
return r.status, r.read()
|
||||
except (OSError, http.client.HTTPException):
|
||||
_local.conn = None
|
||||
raise
|
||||
|
||||
|
||||
def obj(key):
|
||||
return '/' + BUCKET + '/' + key
|
||||
|
||||
|
||||
def list_keys(prefix, identity='api'):
|
||||
keys, token = [], None
|
||||
while True:
|
||||
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
|
||||
if token:
|
||||
q['continuation-token'] = token
|
||||
st, body = request('GET', '/' + BUCKET, identity, query=q)
|
||||
if st != 200:
|
||||
raise SystemExit('list failed: HTTP %d' % st)
|
||||
root = ET.fromstring(body)
|
||||
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
|
||||
if root.findtext(NS + 'IsTruncated') != 'true':
|
||||
return keys
|
||||
token = root.findtext(NS + 'NextContinuationToken')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
|
||||
p.add_argument('arg', nargs='?', default='')
|
||||
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
|
||||
p.add_argument('--size', type=int, default=100)
|
||||
p.add_argument('--workers', type=int, default=32)
|
||||
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
|
||||
a = p.parse_args()
|
||||
out = {'cmd': a.cmd, 'as': a.identity}
|
||||
if a.cmd == 'get':
|
||||
st, body = request('GET', obj(a.arg), a.identity)
|
||||
out.update(status=st, bytes=len(body) if st == 200 else 0,
|
||||
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
|
||||
elif a.cmd == 'put':
|
||||
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
|
||||
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'delete':
|
||||
st, _ = request('DELETE', obj(a.arg), a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'create-bucket':
|
||||
st, _ = request('PUT', '/' + a.arg, a.identity)
|
||||
out.update(status=st)
|
||||
elif a.cmd == 'list':
|
||||
out.update(keys=list_keys(a.arg, a.identity))
|
||||
elif a.cmd == 'count':
|
||||
out.update(count=len(list_keys(a.arg, a.identity)))
|
||||
elif a.cmd in ('bulk-put', 'bulk-delete'):
|
||||
keys = [k.strip() for k in sys.stdin if k.strip()]
|
||||
data = b'x' * a.size
|
||||
|
||||
def one(k):
|
||||
if a.cmd == 'bulk-put':
|
||||
return request('PUT', obj(k), a.identity, body=data)[0]
|
||||
return request('DELETE', obj(k), a.identity)[0]
|
||||
|
||||
start = dt.datetime.now()
|
||||
with ThreadPoolExecutor(a.workers) as pool:
|
||||
codes = list(pool.map(one, keys))
|
||||
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
|
||||
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,331 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/api"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/cache"
|
||||
"mtusi-messenger/internal/db"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/reconcile"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
func getenv(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func getint(key string, def int) int {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n <= 0 {
|
||||
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
|
||||
os.Exit(2)
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func secret(key string) (string, error) {
|
||||
if path := os.Getenv(key + "_FILE"); path != "" {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("read %s_FILE: %w", key, err)
|
||||
}
|
||||
return strings.TrimSpace(string(b)), nil
|
||||
}
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v, nil
|
||||
}
|
||||
return "", fmt.Errorf("%s or %s_FILE is required", key, key)
|
||||
}
|
||||
|
||||
func main() {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
if err := run(log); err != nil {
|
||||
log.Error("fatal", "error", err.Error())
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func openDB(ctx context.Context) (*store.Store, func(), error) {
|
||||
dsn, err := secret("DATABASE_URL")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))})
|
||||
if err != nil {
|
||||
return nil, nil, errors.New("database config is invalid")
|
||||
}
|
||||
return store.New(pool), pool.Close, nil
|
||||
}
|
||||
|
||||
func getduration(key string, def time.Duration) time.Duration {
|
||||
v := os.Getenv(key)
|
||||
if v == "" {
|
||||
return def
|
||||
}
|
||||
d, err := time.ParseDuration(v)
|
||||
if err != nil || d <= 0 {
|
||||
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
|
||||
os.Exit(2)
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") }
|
||||
|
||||
// openObjectsAs reads <prefix>_ACCESS_KEY_ID[_FILE] / <prefix>_SECRET_ACCESS_KEY[_FILE]:
|
||||
// S3 for the restricted API identity, S3_ADMIN only in the one-shot bootstrap.
|
||||
func openObjectsAs(prefix string) (*objstore.Store, error) {
|
||||
access, err := secret(prefix + "_ACCESS_KEY_ID")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return objstore.New(objstore.Config{
|
||||
Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"),
|
||||
Region: getenv("S3_REGION", "us-east-1"),
|
||||
Bucket: getenv("S3_BUCKET", "messenger-attachments"),
|
||||
AccessKey: access,
|
||||
SecretKey: secretKey,
|
||||
})
|
||||
}
|
||||
|
||||
// runReconcile: `server reconcile [--apply] [--grace 15m] [--every 10m]`.
|
||||
// Dry-run by default; prints one JSON report per pass.
|
||||
func runReconcile(ctx context.Context, log *slog.Logger, args []string) error {
|
||||
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
|
||||
apply := fs.Bool("apply", false, "apply fixes (default: dry-run, report only)")
|
||||
grace := fs.Duration("grace", getduration("RECONCILE_GRACE", 15*time.Minute), "minimum age before an object/row is touched")
|
||||
every := fs.Duration("every", 0, "repeat interval; 0 runs once")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return err
|
||||
}
|
||||
if *grace < time.Minute {
|
||||
return errors.New("grace below 1m is unsafe: it is shorter than an upload may take")
|
||||
}
|
||||
st, closeDB, err := openDB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB()
|
||||
objects, err := openObjects()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rec := reconcile.New(st, objects, log)
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
for {
|
||||
pctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
|
||||
rep, err := rec.Run(pctx, reconcile.Options{Apply: *apply, Grace: *grace})
|
||||
cancel()
|
||||
if err != nil {
|
||||
if *every == 0 {
|
||||
return err
|
||||
}
|
||||
log.Warn("reconcile pass failed", "error", err.Error())
|
||||
} else {
|
||||
out, _ := json.Marshal(rep)
|
||||
fmt.Println(string(out))
|
||||
}
|
||||
if *every == 0 {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-stop:
|
||||
return nil
|
||||
case <-time.After(*every):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func run(log *slog.Logger) error {
|
||||
ctx := context.Background()
|
||||
|
||||
if len(os.Args) > 1 && os.Args[1] == "migrate" {
|
||||
dsn, err := secret("DATABASE_URL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2})
|
||||
if err != nil {
|
||||
return errors.New("database config is invalid")
|
||||
}
|
||||
defer pool.Close()
|
||||
mctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
return db.Migrate(mctx, pool, log)
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "s3-init" {
|
||||
objects, err := openObjectsAs("S3_ADMIN")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var created bool
|
||||
for attempt := 1; ; attempt++ {
|
||||
bctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1"))
|
||||
cancel()
|
||||
if err == nil || attempt == 30 {
|
||||
break
|
||||
}
|
||||
log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error())
|
||||
time.Sleep(2 * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("bucket ready", "bucket", objects.Bucket(), "created", created)
|
||||
return nil
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "reconcile" {
|
||||
return runReconcile(ctx, log, os.Args[2:])
|
||||
}
|
||||
|
||||
st, closeDB, err := openDB(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeDB()
|
||||
|
||||
ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second
|
||||
cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second
|
||||
|
||||
var sessCache auth.SessionCache
|
||||
switch mode := getenv("SESSION_STORE", "postgres"); mode {
|
||||
case "postgres":
|
||||
case "valkey", "redis":
|
||||
url, err := secret("CACHE_URL")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vk, err := cache.NewValkey(url)
|
||||
if err != nil {
|
||||
return errors.New("cache config is invalid")
|
||||
}
|
||||
defer vk.Close()
|
||||
sessCache = vk
|
||||
default:
|
||||
return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode)
|
||||
}
|
||||
|
||||
hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4))
|
||||
sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log)
|
||||
|
||||
instanceID := getenv("INSTANCE_ID", "")
|
||||
if instanceID == "" {
|
||||
h, err := os.Hostname()
|
||||
if err != nil || h == "" {
|
||||
return errors.New("cannot determine instance id; set INSTANCE_ID")
|
||||
}
|
||||
instanceID = h
|
||||
}
|
||||
idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second
|
||||
objects, err := openObjects()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760))
|
||||
if maxUpload > 10485760 {
|
||||
return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit")
|
||||
}
|
||||
drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond
|
||||
|
||||
ready := &atomic.Bool{}
|
||||
ready.Store(true)
|
||||
|
||||
host := getenv("APP_HOST", "0.0.0.0")
|
||||
port := getenv("APP_PORT", "8080")
|
||||
addr := host + ":" + port
|
||||
srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{
|
||||
InstanceID: instanceID, IdempotencyTTL: idemTTL,
|
||||
MaxUploadBytes: maxUpload,
|
||||
MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
|
||||
UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)),
|
||||
DecodeBudgetBytes: int64(getint("DECODE_BUDGET_BYTES", 128<<20)),
|
||||
UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second),
|
||||
S3Timeout: getduration("S3_TIMEOUT", 15*time.Second),
|
||||
})
|
||||
httpSrv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: srv.Handler(),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 90 * time.Second,
|
||||
WriteTimeout: 90 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
bgCtx, stopBg := context.WithCancel(ctx)
|
||||
defer stopBg()
|
||||
go func() {
|
||||
t := time.NewTicker(10 * time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-bgCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
c, cancel := context.WithTimeout(bgCtx, 10*time.Second)
|
||||
if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 {
|
||||
log.Info("expired sessions purged", "count", n)
|
||||
}
|
||||
if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 {
|
||||
log.Info("expired idempotency keys purged", "count", n)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- httpSrv.ListenAndServe() }()
|
||||
log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(),
|
||||
"session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds()))
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
case sig := <-stop:
|
||||
log.Info("shutdown signal received", "signal", sig.String())
|
||||
ready.Store(false)
|
||||
time.Sleep(drainDelay)
|
||||
httpSrv.SetKeepAlivesEnabled(false)
|
||||
sctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if err := httpSrv.Shutdown(sctx); err != nil {
|
||||
log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error())
|
||||
_ = httpSrv.Close()
|
||||
}
|
||||
log.Info("server stopped cleanly")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.23
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.7.2
|
||||
github.com/minio/minio-go/v7 v7.0.82
|
||||
github.com/redis/go-redis/v9 v9.7.0
|
||||
golang.org/x/crypto v0.31.0
|
||||
golang.org/x/sync v0.10.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/rs/xid v1.6.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
)
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
|
||||
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
|
||||
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
|
||||
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
|
||||
github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
|
||||
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
|
||||
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
|
||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@@ -0,0 +1,338 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
mimePNG = "image/png"
|
||||
mimeJPEG = "image/jpeg"
|
||||
// An oversized body is read and discarded up to this many bytes so the client
|
||||
// receives 413 instead of a connection reset; beyond it the connection is closed.
|
||||
maxDrainBytes = 64 << 20
|
||||
)
|
||||
|
||||
func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error())
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable"))
|
||||
}
|
||||
|
||||
func validFilename(name string) bool {
|
||||
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." {
|
||||
return false
|
||||
}
|
||||
if n := runeLen(name); n < 1 || n > 255 {
|
||||
return false
|
||||
}
|
||||
for _, c := range name {
|
||||
switch {
|
||||
case c == '/', c == '\\', c == 0, unicode.IsControl(c):
|
||||
return false
|
||||
case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069: // bidi overrides hide the real extension
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func sniffImage(data []byte) string {
|
||||
switch {
|
||||
case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")):
|
||||
return mimePNG
|
||||
case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}):
|
||||
return mimeJPEG
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// decodeCost estimates bytes allocated by a full decode for the memory budget.
|
||||
func decodeCost(m color.Model, pixels int64) int64 {
|
||||
bpp := int64(4)
|
||||
switch m {
|
||||
case color.GrayModel, color.AlphaModel:
|
||||
bpp = 1
|
||||
case color.Gray16Model, color.Alpha16Model:
|
||||
bpp = 2
|
||||
case color.YCbCrModel:
|
||||
bpp = 3
|
||||
case color.RGBA64Model, color.NRGBA64Model:
|
||||
bpp = 8
|
||||
}
|
||||
if _, ok := m.(color.Palette); ok {
|
||||
bpp = 1
|
||||
}
|
||||
return pixels * bpp
|
||||
}
|
||||
|
||||
type imageInfo struct {
|
||||
contentType string
|
||||
width, height int
|
||||
}
|
||||
|
||||
// inspectImage checks the real format by magic bytes, compares it with the
|
||||
// declared Content-Type, enforces the pixel limit from the header only and then
|
||||
// fully decodes the image under a shared memory budget to reject corrupt data.
|
||||
func (s *Server) inspectImage(ctx context.Context, data []byte, declared string) (imageInfo, *apperr.AppError) {
|
||||
detected := sniffImage(data)
|
||||
if detected == "" {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image")
|
||||
}
|
||||
if detected != declared {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content")
|
||||
}
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("image header is invalid")
|
||||
}
|
||||
pixels := int64(cfg.Width) * int64(cfg.Height)
|
||||
if pixels > s.maxPixels {
|
||||
return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels")
|
||||
}
|
||||
cost := min(decodeCost(cfg.ColorModel, pixels), s.decodeBudget)
|
||||
if err := s.decodeSem.Acquire(ctx, cost); err != nil {
|
||||
return imageInfo{}, apperr.Unavailable("server is busy processing images, retry later")
|
||||
}
|
||||
if detected == mimePNG {
|
||||
_, err = png.Decode(bytes.NewReader(data))
|
||||
} else {
|
||||
_, err = jpeg.Decode(bytes.NewReader(data))
|
||||
}
|
||||
s.decodeSem.Release(cost)
|
||||
if err != nil {
|
||||
return imageInfo{}, apperr.UnsupportedMedia("image data is corrupted")
|
||||
}
|
||||
return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil
|
||||
}
|
||||
|
||||
func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) {
|
||||
var buf bytes.Buffer
|
||||
if r.ContentLength > 0 {
|
||||
buf.Grow(int(r.ContentLength))
|
||||
}
|
||||
_, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit))
|
||||
var maxErr *http.MaxBytesError
|
||||
switch {
|
||||
case errors.As(err, &maxErr):
|
||||
return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes")
|
||||
case err != nil:
|
||||
return nil, apperr.InvalidRequest("request body could not be read")
|
||||
case buf.Len() == 0:
|
||||
return nil, apperr.InvalidRequest("request body is empty")
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
body := r.Body
|
||||
defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }()
|
||||
|
||||
id := requestID(r)
|
||||
chatID, ok := s.chatAccess(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
query, err := url.ParseQuery(r.URL.RawQuery)
|
||||
if err != nil || len(query["filename"]) != 1 {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required"))
|
||||
return
|
||||
}
|
||||
filename := query.Get("filename")
|
||||
if !validFilename(filename) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters"))
|
||||
return
|
||||
}
|
||||
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if media != mimePNG && media != mimeJPEG {
|
||||
apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
|
||||
return
|
||||
}
|
||||
if r.ContentLength > s.maxUpload {
|
||||
apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes"))
|
||||
return
|
||||
}
|
||||
|
||||
// Each slot may hold up to maxUpload bytes in memory; waiting is bounded by the request timeout.
|
||||
if err := s.uploadSem.Acquire(r.Context(), 1); err != nil {
|
||||
apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later"))
|
||||
return
|
||||
}
|
||||
defer s.uploadSem.Release(1)
|
||||
|
||||
data, aerr := readBody(w, r, s.maxUpload)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
info, aerr := s.inspectImage(r.Context(), data, media)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
attID := uuid.NewString()
|
||||
att := &model.Attachment{
|
||||
ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename,
|
||||
ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]),
|
||||
Status: "ready", Variants: []model.Variant{},
|
||||
// The key is server-generated; filename never reaches S3.
|
||||
ObjectKey: "attachments/" + chatID + "/" + attID,
|
||||
}
|
||||
|
||||
// 1. intent row (pending) -> 2. S3 PUT -> 3. pending->ready. A crash between
|
||||
// the steps leaves a pending row and maybe an object; both are removed by the
|
||||
// reconciler after the grace period, never an untracked object.
|
||||
if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height); err != nil {
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
|
||||
err = s.objects.Put(putCtx, att.ObjectKey, data, att.ContentType)
|
||||
cancel()
|
||||
if err != nil {
|
||||
go s.compensateUpload(att, id)
|
||||
s.objectError(w, r, err)
|
||||
return
|
||||
}
|
||||
updated, err := s.store.MarkAttachmentReady(r.Context(), att.ID)
|
||||
if err != nil {
|
||||
go s.compensateUpload(att, id)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.Unavailable("upload expired, retry"))
|
||||
return
|
||||
}
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
att.UpdatedAt = model.NewTime(updated)
|
||||
writeJSON(w, r, http.StatusCreated, att)
|
||||
}
|
||||
|
||||
// compensateUpload is best effort and starts from the row: the object is removed
|
||||
// only if this call deleted the still-pending row. If the row is already ready
|
||||
// (the UPDATE committed but its reply was lost) the object is kept. Anything left
|
||||
// behind (pending row while PostgreSQL is down, or an object whose removal
|
||||
// failed) is cleaned up by the reconciler after the grace period.
|
||||
func (s *Server) compensateUpload(att *model.Attachment, reqID string) {
|
||||
dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer dcancel()
|
||||
deleted, err := s.store.DeletePendingAttachment(dctx, att.ID)
|
||||
if err != nil || !deleted {
|
||||
if err != nil {
|
||||
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
|
||||
"attachment_id", att.ID, "stage", "delete_row")
|
||||
}
|
||||
return
|
||||
}
|
||||
octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer ocancel()
|
||||
if err := s.objects.Remove(octx, att.ObjectKey); err != nil {
|
||||
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
|
||||
"attachment_id", att.ID, "stage", "remove_object")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) {
|
||||
id := requestID(r)
|
||||
attID, err := parseUUID(r.PathValue("attachment_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID"))
|
||||
return nil, false
|
||||
}
|
||||
att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible"))
|
||||
return nil, false
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return nil, false
|
||||
}
|
||||
return att, true
|
||||
}
|
||||
|
||||
func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
if att, ok := s.attachmentForRequest(w, r); ok {
|
||||
writeJSON(w, r, http.StatusOK, att)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
att, ok := s.attachmentForRequest(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
switch v := r.URL.Query().Get("variant"); v {
|
||||
case "", "original":
|
||||
case "thumbnail", "crop", "optimized":
|
||||
apperr.Write(w, id, apperr.NotFound("variant not found"))
|
||||
return
|
||||
default:
|
||||
apperr.Write(w, id, apperr.InvalidRequest("unknown variant"))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
|
||||
defer cancel()
|
||||
obj, size, err := s.objects.Get(ctx, att.ObjectKey)
|
||||
if objstore.IsNotFound(err) {
|
||||
s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID)
|
||||
apperr.Write(w, id, apperr.NotFound("attachment content is not available"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.objectError(w, r, err)
|
||||
return
|
||||
}
|
||||
defer obj.Close()
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", att.ContentType)
|
||||
h.Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("Cache-Control", "private, no-store")
|
||||
h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename))
|
||||
h.Set("ETag", `"`+att.SHA256+`"`)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
if _, err := io.Copy(w, obj); err != nil {
|
||||
s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,427 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) {
|
||||
id := requestID(r)
|
||||
if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) {
|
||||
s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error())
|
||||
apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable"))
|
||||
return
|
||||
}
|
||||
s.log.Error("unexpected storage error", "request_id", id, "error", err.Error())
|
||||
apperr.Write(w, id, apperr.Internal("internal error"))
|
||||
}
|
||||
|
||||
func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
|
||||
}
|
||||
|
||||
func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.ready.Load() {
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down"))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
|
||||
defer cancel()
|
||||
if err := s.store.Ping(ctx); err != nil {
|
||||
apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable"))
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
|
||||
}
|
||||
|
||||
type createUserReq struct {
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
func validPassword(p string) bool {
|
||||
n := utf8.RuneCountInString(p)
|
||||
return utf8.ValidString(p) && n >= 12 && n <= 128
|
||||
}
|
||||
|
||||
func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
var req createUserReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
if !usernameRe.MatchString(req.Username) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$"))
|
||||
return
|
||||
}
|
||||
if !validText(req.DisplayName, 100) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points"))
|
||||
return
|
||||
}
|
||||
if !validPassword(req.Password) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points"))
|
||||
return
|
||||
}
|
||||
hash, err := s.hasher.Hash(r.Context(), req.Password)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash)
|
||||
if errors.Is(err, store.ErrUsernameTaken) {
|
||||
apperr.Write(w, id, apperr.Conflict("username is already taken"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, u)
|
||||
}
|
||||
|
||||
func (s *Server) me(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, currentUser(r))
|
||||
}
|
||||
|
||||
const basicChallenge = `Basic realm="messenger", charset="UTF-8"`
|
||||
|
||||
func unauthorizedBasic(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("WWW-Authenticate", basicChallenge)
|
||||
apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password"))
|
||||
}
|
||||
|
||||
func parseBasic(r *http.Request) (string, string, bool) {
|
||||
scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Basic") {
|
||||
return "", "", false
|
||||
}
|
||||
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value))
|
||||
if err != nil || !utf8.Valid(raw) {
|
||||
return "", "", false
|
||||
}
|
||||
user, pass, ok := strings.Cut(string(raw), ":")
|
||||
if !ok || !usernameRe.MatchString(user) || !validPassword(pass) {
|
||||
return "", "", false
|
||||
}
|
||||
return user, pass, true
|
||||
}
|
||||
|
||||
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
username, password, ok := parseBasic(r)
|
||||
if !ok {
|
||||
unauthorizedBasic(w, r)
|
||||
return
|
||||
}
|
||||
user, hash, err := s.store.Credentials(r.Context(), username)
|
||||
if err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
|
||||
valid, err := s.hasher.Verify(r.Context(), password, hash)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if !valid {
|
||||
unauthorizedBasic(w, r)
|
||||
return
|
||||
}
|
||||
token, expires, err := s.sessions.Create(r.Context(), user)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, model.Session{
|
||||
Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodDelete {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
err := s.sessions.Revoke(r.Context(), currentToken(r))
|
||||
if errors.Is(err, auth.ErrInvalidSession) {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("X-Request-Id", requestID(r))
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
type createChatReq struct {
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
}
|
||||
|
||||
func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
s.createChat(w, r)
|
||||
case http.MethodGet:
|
||||
s.listChats(w, r)
|
||||
default:
|
||||
methodNotAllowed(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) createChat(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
var req createChatReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
if !validText(req.Title, 100) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points"))
|
||||
return
|
||||
}
|
||||
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items"))
|
||||
return
|
||||
}
|
||||
seen := make(map[string]bool, len(req.MemberIDs))
|
||||
for _, raw := range req.MemberIDs {
|
||||
uid, err := parseUUID(raw)
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings"))
|
||||
return
|
||||
}
|
||||
if seen[uid] {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id"))
|
||||
return
|
||||
}
|
||||
seen[uid] = true
|
||||
}
|
||||
caller := currentUser(r)
|
||||
if !seen[caller.ID] {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids"))
|
||||
return
|
||||
}
|
||||
|
||||
chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs)
|
||||
if errors.Is(err, store.ErrUnknownMember) {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("unknown member_id"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, chat)
|
||||
}
|
||||
|
||||
func (s *Server) listChats(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
caller := currentUser(r)
|
||||
limit, cur, aerr := pagingParams(r, caller.ID)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
page := model.Page[*model.Chat]{Items: items}
|
||||
if more && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID})
|
||||
page.NextCursor = &encoded
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, r)
|
||||
return
|
||||
}
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return
|
||||
}
|
||||
chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, chat)
|
||||
}
|
||||
|
||||
type createMessageReq struct {
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodPost:
|
||||
s.createMessage(w, r)
|
||||
case http.MethodGet:
|
||||
s.listMessages(w, r)
|
||||
default:
|
||||
methodNotAllowed(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) {
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return "", false
|
||||
}
|
||||
ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return "", false
|
||||
}
|
||||
if !ok {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return "", false
|
||||
}
|
||||
return chatID, true
|
||||
}
|
||||
|
||||
func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
chatID, err := parseUUID(r.PathValue("chat_id"))
|
||||
if err != nil {
|
||||
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
|
||||
return
|
||||
}
|
||||
// Validation errors are reported only to members; outsiders always get 404.
|
||||
reject := func(aerr *apperr.AppError) {
|
||||
if _, ok := s.chatAccess(w, r); !ok {
|
||||
return
|
||||
}
|
||||
apperr.Write(w, id, aerr)
|
||||
}
|
||||
key := r.Header.Get("Idempotency-Key")
|
||||
if !idempotencyKeyRe.MatchString(key) {
|
||||
reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}"))
|
||||
return
|
||||
}
|
||||
var req createMessageReq
|
||||
if aerr := decodeJSON(w, r, &req); aerr != nil {
|
||||
reject(aerr)
|
||||
return
|
||||
}
|
||||
if req.AttachmentIDs == nil {
|
||||
req.AttachmentIDs = []string{}
|
||||
}
|
||||
if len(req.AttachmentIDs) > 10 {
|
||||
reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items"))
|
||||
return
|
||||
}
|
||||
seen := make(map[string]bool, len(req.AttachmentIDs))
|
||||
for _, raw := range req.AttachmentIDs {
|
||||
aid, err := parseUUID(raw)
|
||||
if err != nil {
|
||||
reject(apperr.InvalidRequest("attachment_ids must be UUID strings"))
|
||||
return
|
||||
}
|
||||
if seen[aid] {
|
||||
reject(apperr.InvalidRequest("duplicate attachment_id"))
|
||||
return
|
||||
}
|
||||
seen[aid] = true
|
||||
}
|
||||
if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) {
|
||||
reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments"))
|
||||
return
|
||||
}
|
||||
if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 {
|
||||
reject(apperr.InvalidRequest("text must be at most 4000 code points"))
|
||||
return
|
||||
}
|
||||
msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs,
|
||||
key, messageFingerprint(req), s.idempotencyTTL)
|
||||
if errors.Is(err, store.ErrIdempotencyConflict) {
|
||||
apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
|
||||
return
|
||||
}
|
||||
if errors.Is(err, store.ErrAttachmentNotFound) {
|
||||
apperr.Write(w, id, apperr.NotFound("attachment not found in this chat"))
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
if replayed {
|
||||
w.Header().Set("Idempotent-Replayed", "true")
|
||||
}
|
||||
writeJSON(w, r, http.StatusCreated, msg)
|
||||
}
|
||||
|
||||
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) {
|
||||
id := requestID(r)
|
||||
chatID, ok := s.chatAccess(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, cur, aerr := pagingParams(r, chatID)
|
||||
if aerr != nil {
|
||||
apperr.Write(w, id, aerr)
|
||||
return
|
||||
}
|
||||
items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur)
|
||||
if err != nil {
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
page := model.Page[*model.Message]{Items: items}
|
||||
if more && len(items) > 0 {
|
||||
last := items[len(items)-1]
|
||||
encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID})
|
||||
page.NextCursor = &encoded
|
||||
}
|
||||
writeJSON(w, r, http.StatusOK, page)
|
||||
}
|
||||
|
||||
func (s *Server) notFound(w http.ResponseWriter, r *http.Request) {
|
||||
apperr.Write(w, requestID(r), apperr.NotFound("route not found"))
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
const maxBodyBytes = 1 << 20
|
||||
|
||||
var (
|
||||
usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
)
|
||||
|
||||
func messageFingerprint(req createMessageReq) []byte {
|
||||
b, _ := json.Marshal(req)
|
||||
sum := sha256.Sum256(b)
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
func parseUUID(s string) (string, error) {
|
||||
u, err := uuid.Parse(s)
|
||||
if err != nil || u.String() != s {
|
||||
return "", errors.New("invalid uuid")
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
|
||||
func runeLen(s string) int { return utf8.RuneCountInString(s) }
|
||||
|
||||
func validText(s string, max int) bool {
|
||||
n := runeLen(s)
|
||||
return n >= 1 && n <= max && strings.TrimSpace(s) != ""
|
||||
}
|
||||
|
||||
func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError {
|
||||
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if media != "application/json" {
|
||||
return apperr.UnsupportedMedia("Content-Type must be application/json")
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
|
||||
dec := json.NewDecoder(r.Body)
|
||||
dec.DisallowUnknownFields()
|
||||
if err := dec.Decode(dst); err != nil {
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
return apperr.PayloadTooLarge("request body too large")
|
||||
}
|
||||
return apperr.InvalidRequest("malformed JSON body")
|
||||
}
|
||||
var extra json.RawMessage
|
||||
if err := dec.Decode(&extra); err != io.EOF {
|
||||
var maxErr *http.MaxBytesError
|
||||
if errors.As(err, &maxErr) {
|
||||
return apperr.PayloadTooLarge("request body too large")
|
||||
}
|
||||
return apperr.InvalidRequest("malformed JSON body")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) {
|
||||
limit := 50
|
||||
if v := r.URL.Query().Get("limit"); v != "" {
|
||||
n, err := strconv.Atoi(v)
|
||||
if err != nil || n < 1 || n > 100 {
|
||||
return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100")
|
||||
}
|
||||
limit = n
|
||||
}
|
||||
var cur *store.Cursor
|
||||
if v := r.URL.Query().Get("cursor"); v != "" {
|
||||
c, err := store.DecodeCursor(v)
|
||||
if err == nil {
|
||||
_, err = parseUUID(c.ID)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, nil, apperr.InvalidRequest("invalid cursor")
|
||||
}
|
||||
if c.Scope != scope {
|
||||
return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource")
|
||||
}
|
||||
cur = &c
|
||||
}
|
||||
return limit, cur, nil
|
||||
}
|
||||
|
||||
func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("X-Request-Id", requestID(r))
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(payload)
|
||||
}
|
||||
|
||||
func methodNotAllowed(w http.ResponseWriter, r *http.Request) {
|
||||
apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed"))
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi-messenger/internal/apperr"
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
type ctxKey int
|
||||
|
||||
const (
|
||||
ctxRequestID ctxKey = iota
|
||||
ctxUser
|
||||
ctxToken
|
||||
)
|
||||
|
||||
func requestIDMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.Header.Get("X-Request-Id")
|
||||
if id == "" {
|
||||
id = uuid.NewString()
|
||||
}
|
||||
w.Header().Set("X-Request-Id", id)
|
||||
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id)))
|
||||
})
|
||||
}
|
||||
|
||||
func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Instance-Id", instanceID)
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func requestID(r *http.Request) string {
|
||||
if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" {
|
||||
return id
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
func (s *statusWriter) WriteHeader(code int) {
|
||||
s.status = code
|
||||
s.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
|
||||
next.ServeHTTP(sw, r)
|
||||
|
||||
log.Info("http",
|
||||
"method", r.Method,
|
||||
"path", r.URL.Path,
|
||||
"status", sw.status,
|
||||
"dur_ms", time.Since(start).Milliseconds(),
|
||||
"request_id", requestID(r),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r))
|
||||
apperr.Write(w, requestID(r), apperr.Internal("internal error"))
|
||||
}
|
||||
}()
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// Binary transfers get a longer budget than JSON requests.
|
||||
func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
limit := d
|
||||
if strings.Contains(r.URL.Path, "/attachments") {
|
||||
limit = binary
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), limit)
|
||||
defer cancel()
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
}
|
||||
|
||||
const bearerChallenge = `Bearer realm="messenger"`
|
||||
|
||||
func unauthorizedBearer(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("WWW-Authenticate", bearerChallenge)
|
||||
apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required"))
|
||||
}
|
||||
|
||||
func bearerToken(r *http.Request) (string, bool) {
|
||||
scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ")
|
||||
if !ok || !strings.EqualFold(scheme, "Bearer") {
|
||||
return "", false
|
||||
}
|
||||
token = strings.TrimSpace(token)
|
||||
if token == "" || len(token) > 256 {
|
||||
return "", false
|
||||
}
|
||||
return token, true
|
||||
}
|
||||
|
||||
func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
token, ok := bearerToken(r)
|
||||
if !ok {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
user, err := s.sessions.Resolve(r.Context(), token)
|
||||
if errors.Is(err, auth.ErrInvalidSession) {
|
||||
unauthorizedBearer(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
|
||||
s.storeError(w, r, err)
|
||||
return
|
||||
}
|
||||
ctx := context.WithValue(r.Context(), ctxUser, user)
|
||||
ctx = context.WithValue(ctx, ctxToken, token)
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
}
|
||||
}
|
||||
|
||||
func currentUser(r *http.Request) *model.User {
|
||||
return r.Context().Value(ctxUser).(*model.User)
|
||||
}
|
||||
|
||||
func currentToken(r *http.Request) string {
|
||||
return r.Context().Value(ctxToken).(string)
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sync/semaphore"
|
||||
|
||||
"mtusi-messenger/internal/auth"
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
store *store.Store
|
||||
objects *objstore.Store
|
||||
sessions *auth.Sessions
|
||||
hasher *auth.Hasher
|
||||
log *slog.Logger
|
||||
ready *atomic.Bool
|
||||
timeout time.Duration
|
||||
|
||||
instanceID string
|
||||
idempotencyTTL time.Duration
|
||||
|
||||
uploadTimeout time.Duration
|
||||
s3Timeout time.Duration
|
||||
maxUpload int64
|
||||
maxPixels int64
|
||||
decodeBudget int64
|
||||
uploadSem *semaphore.Weighted
|
||||
decodeSem *semaphore.Weighted
|
||||
}
|
||||
|
||||
type Options struct {
|
||||
InstanceID string
|
||||
IdempotencyTTL time.Duration
|
||||
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
UploadConcurrency int64
|
||||
DecodeBudgetBytes int64
|
||||
UploadTimeout time.Duration
|
||||
S3Timeout time.Duration
|
||||
}
|
||||
|
||||
func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server {
|
||||
return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second,
|
||||
instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL,
|
||||
uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout,
|
||||
maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels, decodeBudget: opt.DecodeBudgetBytes,
|
||||
uploadSem: semaphore.NewWeighted(opt.UploadConcurrency), decodeSem: semaphore.NewWeighted(opt.DecodeBudgetBytes)}
|
||||
}
|
||||
|
||||
func (s *Server) Handler() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/health/live", s.healthLive)
|
||||
mux.HandleFunc("/health/ready", s.healthReady)
|
||||
mux.HandleFunc("/api/v1/users", s.usersRoute)
|
||||
mux.HandleFunc("/api/v1/auth/sessions", s.createSession)
|
||||
mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession))
|
||||
mux.HandleFunc("/api/v1/users/me", s.auth(s.me))
|
||||
mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute))
|
||||
mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment))
|
||||
mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment))
|
||||
mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment))
|
||||
mux.HandleFunc("/", s.notFound)
|
||||
|
||||
var h http.Handler = mux
|
||||
h = timeoutMiddleware(s.timeout, s.uploadTimeout, h)
|
||||
h = recoverMiddleware(s.log, h)
|
||||
h = loggingMiddleware(s.log, h)
|
||||
h = requestIDMiddleware(h)
|
||||
h = instanceIDMiddleware(s.instanceID, h)
|
||||
return h
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package apperr
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type Code string
|
||||
|
||||
const (
|
||||
CodeInvalidRequest Code = "invalid_request"
|
||||
CodeUnauthorized Code = "unauthorized"
|
||||
CodeNotFound Code = "not_found"
|
||||
CodeConflict Code = "conflict"
|
||||
CodePayloadTooLarge Code = "payload_too_large"
|
||||
CodeUnsupportedMedia Code = "unsupported_media_type"
|
||||
CodeRateLimited Code = "rate_limited"
|
||||
CodeUnavailable Code = "unavailable"
|
||||
CodeInternal Code = "internal_error"
|
||||
)
|
||||
|
||||
type Body struct {
|
||||
Code Code `json:"code"`
|
||||
Message string `json:"message"`
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
type Envelope struct {
|
||||
Error Body `json:"error"`
|
||||
}
|
||||
|
||||
type AppError struct {
|
||||
Status int
|
||||
Code Code
|
||||
Msg string
|
||||
}
|
||||
|
||||
func (e *AppError) Error() string { return e.Msg }
|
||||
|
||||
func New(status int, code Code, msg string) *AppError {
|
||||
return &AppError{Status: status, Code: code, Msg: msg}
|
||||
}
|
||||
|
||||
func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) }
|
||||
func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) }
|
||||
func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) }
|
||||
func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) }
|
||||
func PayloadTooLarge(msg string) *AppError {
|
||||
return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg)
|
||||
}
|
||||
func UnsupportedMedia(msg string) *AppError {
|
||||
return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg)
|
||||
}
|
||||
func Unavailable(msg string) *AppError {
|
||||
return New(http.StatusServiceUnavailable, CodeUnavailable, msg)
|
||||
}
|
||||
func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) }
|
||||
|
||||
func Write(w http.ResponseWriter, requestID string, err *AppError) {
|
||||
w.Header().Set("Content-Type", "application/json; charset=utf-8")
|
||||
w.Header().Set("X-Request-Id", requestID)
|
||||
w.WriteHeader(err.Status)
|
||||
_ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}})
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
const (
|
||||
argonMemoryKiB = 19 * 1024
|
||||
argonTime = 2
|
||||
argonThreads = 1
|
||||
argonKeyLen = 32
|
||||
argonSaltLen = 16
|
||||
)
|
||||
|
||||
var ErrBadHash = errors.New("malformed password hash")
|
||||
|
||||
type Hasher struct {
|
||||
slots chan struct{}
|
||||
dummy string
|
||||
}
|
||||
|
||||
func NewHasher(concurrency int) *Hasher {
|
||||
h := &Hasher{slots: make(chan struct{}, concurrency)}
|
||||
|
||||
salt := make([]byte, argonSaltLen)
|
||||
_, _ = rand.Read(salt)
|
||||
h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen))
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *Hasher) acquire(ctx context.Context) error {
|
||||
select {
|
||||
case h.slots <- struct{}{}:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Hasher) release() { <-h.slots }
|
||||
|
||||
func encode(salt, key []byte) string {
|
||||
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads,
|
||||
base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key))
|
||||
}
|
||||
|
||||
func (h *Hasher) Hash(ctx context.Context, password string) (string, error) {
|
||||
if err := h.acquire(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer h.release()
|
||||
salt := make([]byte, argonSaltLen)
|
||||
if _, err := rand.Read(salt); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil
|
||||
}
|
||||
|
||||
func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) {
|
||||
missing := encoded == ""
|
||||
if missing {
|
||||
encoded = h.dummy
|
||||
}
|
||||
parts := strings.Split(encoded, "$")
|
||||
if len(parts) != 6 || parts[1] != "argon2id" {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
var version int
|
||||
var mem uint32
|
||||
var t uint32
|
||||
var p uint8
|
||||
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
|
||||
if err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
want, err := base64.RawStdEncoding.DecodeString(parts[5])
|
||||
if err != nil {
|
||||
return false, ErrBadHash
|
||||
}
|
||||
if err := h.acquire(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want)))
|
||||
h.release()
|
||||
return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package auth
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrInvalidSession = errors.New("invalid session")
|
||||
ErrCacheUnavailable = errors.New("session cache unavailable")
|
||||
)
|
||||
|
||||
type SessionCache interface {
|
||||
Get(ctx context.Context, key string) (*CachedSession, error)
|
||||
Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error
|
||||
Del(ctx context.Context, key string) error
|
||||
}
|
||||
|
||||
type CachedSession struct {
|
||||
User model.User `json:"user"`
|
||||
ExpiresAt time.Time `json:"exp"`
|
||||
}
|
||||
|
||||
type Sessions struct {
|
||||
store *store.Store
|
||||
cache SessionCache
|
||||
ttl time.Duration
|
||||
cacheMaxTTL time.Duration
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions {
|
||||
return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log}
|
||||
}
|
||||
|
||||
func NewToken() (string, []byte, error) {
|
||||
raw := make([]byte, 32)
|
||||
if _, err := rand.Read(raw); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
token := base64.RawURLEncoding.EncodeToString(raw)
|
||||
return token, HashToken(token), nil
|
||||
}
|
||||
|
||||
func HashToken(token string) []byte {
|
||||
sum := sha256.Sum256([]byte(token))
|
||||
return sum[:]
|
||||
}
|
||||
|
||||
func cacheKey(hash []byte) string {
|
||||
return "sess:" + base64.RawURLEncoding.EncodeToString(hash)
|
||||
}
|
||||
|
||||
func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) {
|
||||
token, hash, err := NewToken()
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl)
|
||||
if err != nil {
|
||||
return "", time.Time{}, err
|
||||
}
|
||||
s.cacheSet(ctx, hash, user, expires)
|
||||
return token, expires, nil
|
||||
}
|
||||
|
||||
func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) {
|
||||
hash := HashToken(token)
|
||||
if s.cache != nil {
|
||||
cs, err := s.cache.Get(ctx, cacheKey(hash))
|
||||
if err == nil && cs != nil {
|
||||
if time.Now().Before(cs.ExpiresAt) {
|
||||
u := cs.User
|
||||
return &u, nil
|
||||
}
|
||||
|
||||
} else if err != nil {
|
||||
s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error())
|
||||
}
|
||||
}
|
||||
user, expires, err := s.store.SessionUser(ctx, hash)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, ErrInvalidSession
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.cacheSet(ctx, hash, user, expires)
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func (s *Sessions) Revoke(ctx context.Context, token string) error {
|
||||
hash := HashToken(token)
|
||||
if s.cache != nil {
|
||||
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
|
||||
return ErrCacheUnavailable
|
||||
}
|
||||
}
|
||||
err := s.store.RevokeSession(ctx, hash)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return ErrInvalidSession
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s.cache != nil {
|
||||
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
|
||||
|
||||
s.log.Warn("session cache delete after revoke failed", "error", err.Error())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) {
|
||||
if s.cache == nil {
|
||||
return
|
||||
}
|
||||
ttl := time.Until(expires)
|
||||
if ttl > s.cacheMaxTTL {
|
||||
ttl = s.cacheMaxTTL
|
||||
}
|
||||
if ttl <= 0 {
|
||||
return
|
||||
}
|
||||
if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil {
|
||||
s.log.Debug("session cache set failed", "error", err.Error())
|
||||
}
|
||||
}
|
||||
Vendored
+80
@@ -0,0 +1,80 @@
|
||||
package cache
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"mtusi-messenger/internal/auth"
|
||||
)
|
||||
|
||||
var ErrOpen = errors.New("cache circuit open")
|
||||
|
||||
type Valkey struct {
|
||||
rdb *redis.Client
|
||||
openUntil atomic.Int64
|
||||
cooldown time.Duration
|
||||
}
|
||||
|
||||
func NewValkey(url string) (*Valkey, error) {
|
||||
opt, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
opt.DialTimeout = 500 * time.Millisecond
|
||||
opt.ReadTimeout = 300 * time.Millisecond
|
||||
opt.WriteTimeout = 300 * time.Millisecond
|
||||
opt.PoolSize = 10
|
||||
opt.MaxRetries = 0
|
||||
return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil
|
||||
}
|
||||
|
||||
func (v *Valkey) Close() error { return v.rdb.Close() }
|
||||
|
||||
func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) }
|
||||
|
||||
func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() }
|
||||
|
||||
func (v *Valkey) guard(err error) error {
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) {
|
||||
if v.open() {
|
||||
return nil, ErrOpen
|
||||
}
|
||||
raw, err := v.rdb.Get(ctx, key).Bytes()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err := v.guard(err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cs auth.CachedSession
|
||||
if err := json.Unmarshal(raw, &cs); err != nil {
|
||||
return nil, nil
|
||||
}
|
||||
return &cs, nil
|
||||
}
|
||||
|
||||
func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error {
|
||||
if v.open() {
|
||||
return ErrOpen
|
||||
}
|
||||
raw, err := json.Marshal(s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err())
|
||||
}
|
||||
|
||||
func (v *Valkey) Del(ctx context.Context, key string) error {
|
||||
return v.guard(v.rdb.Del(ctx, key).Err())
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
//go:embed migrations/*.sql
|
||||
var migrationFS embed.FS
|
||||
|
||||
const advisoryLockID = 20260202
|
||||
|
||||
type migration struct {
|
||||
version int
|
||||
name string
|
||||
sql string
|
||||
}
|
||||
|
||||
func loadMigrations() ([]migration, error) {
|
||||
entries, err := fs.ReadDir(migrationFS, "migrations")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var out []migration
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
prefix, _, ok := strings.Cut(name, "_")
|
||||
if !ok || !strings.HasSuffix(name, ".sql") {
|
||||
return nil, fmt.Errorf("bad migration file name %q", name)
|
||||
}
|
||||
v, err := strconv.Atoi(prefix)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bad migration version in %q", name)
|
||||
}
|
||||
body, err := migrationFS.ReadFile("migrations/" + name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, migration{version: v, name: name, sql: string(body)})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
|
||||
for i := 1; i < len(out); i++ {
|
||||
if out[i].version == out[i-1].version {
|
||||
return nil, fmt.Errorf("duplicate migration version %d", out[i].version)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error {
|
||||
migs, err := loadMigrations()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
conn, err := pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Release()
|
||||
|
||||
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)
|
||||
|
||||
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version integer PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())`); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
applied := map[int]bool{}
|
||||
rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var v int
|
||||
if err := rows.Scan(&v); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
applied[v] = true
|
||||
}
|
||||
rows.Close()
|
||||
if err := rows.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, m := range migs {
|
||||
if applied[m.version] {
|
||||
continue
|
||||
}
|
||||
tx, err := conn.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, m.sql); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return fmt.Errorf("migration %s: %w", m.name, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil {
|
||||
_ = tx.Rollback(ctx)
|
||||
return err
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("migration applied", "version", m.version, "name", m.name)
|
||||
}
|
||||
log.Info("migrations up to date", "total", len(migs))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT users_username_key UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE
|
||||
);
|
||||
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
|
||||
CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,13 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idem_key),
|
||||
CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32)
|
||||
);
|
||||
CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at);
|
||||
@@ -0,0 +1,51 @@
|
||||
-- Lab4: attachment metadata lives here, bytes live in the private S3 bucket.
|
||||
-- Text may be empty when a message carries at least one attachment (checked by the API).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
|
||||
-- status: pending = intent row written before S3 PUT (not visible through the API);
|
||||
-- ready = object stored and metadata committed.
|
||||
-- missing_at is set by the reconciler when a ready row has no S3 object.
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
sha256 text NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
status text NOT NULL,
|
||||
missing_at timestamptz,
|
||||
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
|
||||
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
|
||||
CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id),
|
||||
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready'))
|
||||
);
|
||||
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
|
||||
|
||||
-- (chat_id, attachment_id) -> attachments (chat_id, id): the database itself refuses
|
||||
-- to link a file of another chat to a message.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
|
||||
DEFERRABLE INITIALLY DEFERRED,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id)
|
||||
REFERENCES attachments (chat_id, id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,31 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type PoolOptions struct {
|
||||
MaxConns int32
|
||||
}
|
||||
|
||||
func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) {
|
||||
cfg, err := pgxpool.ParseConfig(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.MaxConns = opt.MaxConns
|
||||
cfg.MinConns = 1
|
||||
cfg.MaxConnLifetime = 30 * time.Minute
|
||||
cfg.MaxConnIdleTime = 5 * time.Minute
|
||||
cfg.HealthCheckPeriod = 15 * time.Second
|
||||
cfg.ConnConfig.ConnectTimeout = 3 * time.Second
|
||||
rp := cfg.ConnConfig.RuntimeParams
|
||||
rp["application_name"] = "mtusi-messenger"
|
||||
rp["statement_timeout"] = "5000"
|
||||
rp["lock_timeout"] = "2000"
|
||||
rp["idle_in_transaction_session_timeout"] = "10000"
|
||||
return pgxpool.NewWithConfig(ctx, cfg)
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package model
|
||||
|
||||
import "time"
|
||||
|
||||
type Time struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
func NewTime(t time.Time) Time { return Time{Time: t} }
|
||||
|
||||
func (t Time) MarshalJSON() ([]byte, error) {
|
||||
return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil
|
||||
}
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id"`
|
||||
Username string `json:"username"`
|
||||
DisplayName string `json:"display_name"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Chat struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
MemberIDs []string `json:"member_ids"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt Time `json:"created_at"`
|
||||
UpdatedAt Time `json:"updated_at"`
|
||||
|
||||
ObjectKey string `json:"-"`
|
||||
Missing bool `json:"-"`
|
||||
}
|
||||
|
||||
type Page[T any] struct {
|
||||
Items []T `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type Health struct {
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
type Session struct {
|
||||
Token string `json:"token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresAt Time `json:"expires_at"`
|
||||
User User `json:"user"`
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
package objstore
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
}
|
||||
|
||||
type Store struct {
|
||||
c *minio.Client
|
||||
bucket string
|
||||
}
|
||||
|
||||
type Object struct {
|
||||
Key string
|
||||
Size int64
|
||||
LastModified time.Time
|
||||
}
|
||||
|
||||
func New(cfg Config) (*Store, error) {
|
||||
u, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
||||
return nil, errors.New("S3_ENDPOINT must be http(s)://host:port")
|
||||
}
|
||||
if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
|
||||
return nil, errors.New("S3 bucket and credentials are required")
|
||||
}
|
||||
// The SDK retries 10 times with backoff by default; a dead S3 must turn into a
|
||||
// fast 503 instead (the client may retry the upload itself).
|
||||
minio.MaxRetry = 1
|
||||
transport := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
|
||||
MaxIdleConnsPerHost: 32,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
}
|
||||
c, err := minio.New(u.Host, &minio.Options{
|
||||
Transport: transport,
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: u.Scheme == "https",
|
||||
Region: cfg.Region,
|
||||
BucketLookup: minio.BucketLookupPath,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Store{c: c, bucket: cfg.Bucket}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Bucket() string { return s.bucket }
|
||||
|
||||
// Ping checks that the bucket is reachable with the configured credentials.
|
||||
func (s *Store) Ping(ctx context.Context) error {
|
||||
ok, err := s.c.BucketExists(ctx, s.bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("bucket %q does not exist", s.bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnsureBucket creates the bucket if it is absent; safe to repeat.
|
||||
func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) {
|
||||
ok, err := s.c.BucketExists(ctx, s.bucket)
|
||||
if err != nil || ok {
|
||||
return false, err
|
||||
}
|
||||
if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil {
|
||||
if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error {
|
||||
_, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true})
|
||||
return err
|
||||
}
|
||||
|
||||
// Get returns a stream of the object; Stat is called first so that a missing
|
||||
// object or an unavailable S3 is reported before any response byte is written.
|
||||
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
|
||||
obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, 0, err
|
||||
}
|
||||
return obj, info.Size, nil
|
||||
}
|
||||
|
||||
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
|
||||
info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return Object{}, err
|
||||
}
|
||||
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
|
||||
}
|
||||
|
||||
func (s *Store) Remove(ctx context.Context, key string) error {
|
||||
return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
|
||||
}
|
||||
|
||||
func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error {
|
||||
for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
|
||||
if info.Err != nil {
|
||||
return info.Err
|
||||
}
|
||||
if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func IsNotFound(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
switch minio.ToErrorResponse(err).Code {
|
||||
case "NoSuchKey", "NotFound":
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
// Package reconcile compares attachment metadata in PostgreSQL with objects in
|
||||
// the S3 bucket and repairs what can be repaired safely.
|
||||
//
|
||||
// Order matters: S3 is listed first, the database is read second. An upload
|
||||
// writes its pending row before the PUT, so an object seen in the listing
|
||||
// already has its row when the database is read. Every destructive decision is
|
||||
// re-checked against the source of truth right before it is applied.
|
||||
package reconcile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/internal/objstore"
|
||||
"mtusi-messenger/internal/store"
|
||||
)
|
||||
|
||||
const (
|
||||
Prefix = "attachments/"
|
||||
lockID = 20260404
|
||||
)
|
||||
|
||||
type Options struct {
|
||||
Apply bool
|
||||
Grace time.Duration
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
type Item struct {
|
||||
Kind string `json:"kind"`
|
||||
Key string `json:"key,omitempty"`
|
||||
ID string `json:"attachment_id,omitempty"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Mode string `json:"mode"`
|
||||
GraceSeconds int64 `json:"grace_seconds"`
|
||||
ObjectsScanned int `json:"objects_scanned"`
|
||||
RowsScanned int `json:"rows_scanned"`
|
||||
OrphanObjects int `json:"orphan_objects"`
|
||||
FreshSkipped int `json:"fresh_skipped"`
|
||||
StalePending int `json:"stale_pending"`
|
||||
MissingObjects int `json:"missing_objects"`
|
||||
SizeMismatch int `json:"size_mismatch"`
|
||||
Recovered int `json:"recovered"`
|
||||
Fixed int `json:"fixed"`
|
||||
Errors int `json:"errors"`
|
||||
ListMS int64 `json:"list_ms"`
|
||||
DBMS int64 `json:"db_ms"`
|
||||
TotalMS int64 `json:"total_ms"`
|
||||
Items []Item `json:"items"`
|
||||
Truncated bool `json:"items_truncated"`
|
||||
PerObjectUS float64 `json:"per_object_us"`
|
||||
}
|
||||
|
||||
var ErrLocked = errors.New("another reconciler run holds the lock")
|
||||
|
||||
type Reconciler struct {
|
||||
db *store.Store
|
||||
s3 *objstore.Store
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func New(db *store.Store, s3 *objstore.Store, log *slog.Logger) *Reconciler {
|
||||
return &Reconciler{db: db, s3: s3, log: log}
|
||||
}
|
||||
|
||||
func (r *Reconciler) note(rep *Report, it Item) {
|
||||
if len(rep.Items) < 200 {
|
||||
rep.Items = append(rep.Items, it)
|
||||
} else {
|
||||
rep.Truncated = true
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) Run(ctx context.Context, opt Options) (*Report, error) {
|
||||
if opt.Now == nil {
|
||||
opt.Now = time.Now
|
||||
}
|
||||
unlock, ok, err := r.db.TryLock(ctx, lockID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, ErrLocked
|
||||
}
|
||||
defer unlock()
|
||||
|
||||
rep := &Report{Mode: "dry-run", GraceSeconds: int64(opt.Grace.Seconds()), Items: []Item{}}
|
||||
if opt.Apply {
|
||||
rep.Mode = "apply"
|
||||
}
|
||||
start := time.Now()
|
||||
cutoff := opt.Now().Add(-opt.Grace)
|
||||
|
||||
objects := make(map[string]objstore.Object)
|
||||
if err := r.s3.List(ctx, Prefix, func(o objstore.Object) error {
|
||||
objects[o.Key] = o
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rep.ObjectsScanned = len(objects)
|
||||
rep.ListMS = time.Since(start).Milliseconds()
|
||||
|
||||
dbStart := time.Now()
|
||||
known := make(map[string]bool, len(objects))
|
||||
var refs []store.AttachmentRef
|
||||
if err := r.db.EachAttachmentRef(ctx, func(a store.AttachmentRef) error {
|
||||
known[a.ObjectKey] = true
|
||||
rep.RowsScanned++
|
||||
obj, present := objects[a.ObjectKey]
|
||||
switch {
|
||||
case a.Status == "pending" && a.CreatedAt.Before(cutoff):
|
||||
refs = append(refs, a)
|
||||
case a.Status == "ready" && !present && !a.Missing && a.CreatedAt.Before(cutoff):
|
||||
refs = append(refs, a)
|
||||
case a.Status == "ready" && present && a.Missing:
|
||||
refs = append(refs, a)
|
||||
case a.Status == "ready" && present && obj.Size != a.SizeBytes:
|
||||
rep.SizeMismatch++
|
||||
r.note(rep, Item{Kind: "size_mismatch", Key: a.ObjectKey, ID: a.ID, Action: "report only"})
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rep.DBMS = time.Since(dbStart).Milliseconds()
|
||||
|
||||
for _, a := range refs {
|
||||
_, present := objects[a.ObjectKey]
|
||||
switch {
|
||||
case a.Status == "pending":
|
||||
r.stalePending(ctx, rep, a, opt.Apply)
|
||||
case !present:
|
||||
r.missing(ctx, rep, a, opt.Apply)
|
||||
default:
|
||||
r.recovered(ctx, rep, a, opt.Apply)
|
||||
}
|
||||
}
|
||||
|
||||
for key, o := range objects {
|
||||
if known[key] {
|
||||
continue
|
||||
}
|
||||
if !o.LastModified.Before(cutoff) {
|
||||
rep.FreshSkipped++
|
||||
r.note(rep, Item{Kind: "orphan_object", Key: key, Action: "skip: younger than grace"})
|
||||
continue
|
||||
}
|
||||
r.orphan(ctx, rep, key, opt.Apply)
|
||||
}
|
||||
|
||||
rep.TotalMS = time.Since(start).Milliseconds()
|
||||
if rep.ObjectsScanned > 0 {
|
||||
rep.PerObjectUS = float64(time.Since(start).Microseconds()) / float64(rep.ObjectsScanned)
|
||||
}
|
||||
return rep, nil
|
||||
}
|
||||
|
||||
func (r *Reconciler) fail(rep *Report, it Item, err error) {
|
||||
rep.Errors++
|
||||
it.Action = "error: " + strings.SplitN(err.Error(), "\n", 2)[0]
|
||||
r.note(rep, it)
|
||||
}
|
||||
|
||||
// Stale pending row: the upload never reached "ready", so no message can
|
||||
// reference it. The row is deleted first (conditionally), then the object.
|
||||
func (r *Reconciler) stalePending(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
|
||||
rep.StalePending++
|
||||
it := Item{Kind: "stale_pending", Key: a.ObjectKey, ID: a.ID, Action: "would delete row and object"}
|
||||
if !apply {
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
deleted, err := r.db.DeletePendingAttachment(ctx, a.ID)
|
||||
if err != nil {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
if !deleted {
|
||||
it.Action = "skip: row changed concurrently"
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
if err := r.s3.Remove(ctx, a.ObjectKey); err != nil {
|
||||
// The object becomes an orphan and is removed by the next run.
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
rep.Fixed++
|
||||
it.Action = "deleted row and object"
|
||||
r.note(rep, it)
|
||||
}
|
||||
|
||||
// Orphan object: no metadata row. The DB is re-checked right before deletion.
|
||||
func (r *Reconciler) orphan(ctx context.Context, rep *Report, key string, apply bool) {
|
||||
rep.OrphanObjects++
|
||||
it := Item{Kind: "orphan_object", Key: key, Action: "would delete object"}
|
||||
if !apply {
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
known, err := r.db.ObjectKeyKnown(ctx, key)
|
||||
if err != nil {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
if known {
|
||||
it.Action = "skip: row appeared concurrently"
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
if err := r.s3.Remove(ctx, key); err != nil {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
rep.Fixed++
|
||||
it.Action = "deleted object"
|
||||
r.note(rep, it)
|
||||
}
|
||||
|
||||
// Missing object of a ready row: bytes cannot be restored, metadata of a
|
||||
// possibly referenced attachment is never deleted. The row is marked so that
|
||||
// operators see it; StatObject confirms the absence first.
|
||||
func (r *Reconciler) missing(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
|
||||
_, err := r.s3.Stat(ctx, a.ObjectKey)
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
it := Item{Kind: "missing_object", Key: a.ObjectKey, ID: a.ID, Action: "would mark missing (metadata kept)"}
|
||||
if !objstore.IsNotFound(err) {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
rep.MissingObjects++
|
||||
if !apply {
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
if err := r.db.SetAttachmentMissing(ctx, a.ID, true); err != nil {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
rep.Fixed++
|
||||
it.Action = "marked missing (metadata kept)"
|
||||
r.note(rep, it)
|
||||
}
|
||||
|
||||
func (r *Reconciler) recovered(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
|
||||
rep.Recovered++
|
||||
it := Item{Kind: "recovered_object", Key: a.ObjectKey, ID: a.ID, Action: "would clear missing mark"}
|
||||
if !apply {
|
||||
r.note(rep, it)
|
||||
return
|
||||
}
|
||||
if err := r.db.SetAttachmentMissing(ctx, a.ID, false); err != nil {
|
||||
r.fail(rep, it, err)
|
||||
return
|
||||
}
|
||||
rep.Fixed++
|
||||
it.Action = "cleared missing mark"
|
||||
r.note(rep, it)
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.object_key, a.missing_at IS NOT NULL, a.created_at, a.updated_at`
|
||||
|
||||
func scanAttachment(row pgx.Row) (*model.Attachment, error) {
|
||||
a := &model.Attachment{Status: "ready", Variants: []model.Variant{}}
|
||||
var created, updated time.Time
|
||||
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.ObjectKey, &a.Missing, &created, &updated)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.CreatedAt, a.UpdatedAt = utc(created), utc(updated)
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// InsertPendingAttachment records the intent before the S3 PUT. A pending row is
|
||||
// invisible to the API; if the upload never completes, the reconciler removes
|
||||
// the row and the object after the grace period.
|
||||
func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int) error {
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, object_key, status)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'pending')
|
||||
RETURNING created_at`,
|
||||
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
|
||||
width, height, a.ObjectKey).Scan(&created)
|
||||
if pgCode(err) == "23503" {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
a.CreatedAt = utc(created)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MarkAttachmentReady succeeds only while the row is still pending: if the
|
||||
// reconciler already removed a stale intent, the upload must not report 201.
|
||||
func (s *Store) MarkAttachmentReady(ctx context.Context, id string) (time.Time, error) {
|
||||
var updated time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
UPDATE attachments SET status = 'ready', updated_at = date_trunc('milliseconds', now())
|
||||
WHERE id = $1 AND status = 'pending'
|
||||
RETURNING updated_at`, id).Scan(&updated)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return time.Time{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return updated.UTC(), nil
|
||||
}
|
||||
|
||||
// DeletePendingAttachment removes an intent row that never became ready.
|
||||
func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) {
|
||||
tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return tag.RowsAffected() == 1, nil
|
||||
}
|
||||
|
||||
// AttachmentForUser returns a ready attachment only to a member of its chat.
|
||||
func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) {
|
||||
a, err := scanAttachment(s.pool.QueryRow(ctx, `
|
||||
SELECT `+attachmentColumns+`
|
||||
FROM attachments a
|
||||
WHERE a.id = $1 AND a.status = 'ready'
|
||||
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`,
|
||||
id, userID))
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return a, err
|
||||
}
|
||||
|
||||
// AttachmentRef is the reconciler's view of one metadata row.
|
||||
type AttachmentRef struct {
|
||||
ID string
|
||||
ObjectKey string
|
||||
Status string
|
||||
SizeBytes int64
|
||||
Missing bool
|
||||
CreatedAt time.Time
|
||||
Referenced bool
|
||||
}
|
||||
|
||||
func (s *Store) EachAttachmentRef(ctx context.Context, fn func(AttachmentRef) error) error {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT a.id, a.object_key, a.status, a.size_bytes, a.missing_at IS NOT NULL, a.created_at,
|
||||
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
|
||||
FROM attachments a`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var r AttachmentRef
|
||||
if err := rows.Scan(&r.ID, &r.ObjectKey, &r.Status, &r.SizeBytes, &r.Missing, &r.CreatedAt, &r.Referenced); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := fn(r); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) ObjectKeyKnown(ctx context.Context, key string) (bool, error) {
|
||||
var ok bool
|
||||
err := s.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`, key).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (s *Store) SetAttachmentMissing(ctx context.Context, id string, missing bool) error {
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET missing_at = CASE WHEN $2 THEN COALESCE(missing_at, now()) END
|
||||
WHERE id = $1 AND status = 'ready'`, id, missing)
|
||||
return err
|
||||
}
|
||||
|
||||
// TryLock takes a session-level advisory lock on a dedicated connection so that
|
||||
// only one reconciler pass runs at a time. The returned func releases it.
|
||||
func (s *Store) TryLock(ctx context.Context, id int64) (func(), bool, error) {
|
||||
conn, err := s.pool.Acquire(ctx)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
var ok bool
|
||||
if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock($1)`, id).Scan(&ok); err != nil || !ok {
|
||||
conn.Release()
|
||||
return nil, false, err
|
||||
}
|
||||
return func() {
|
||||
_, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, id)
|
||||
conn.Release()
|
||||
}, true, nil
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Cursor struct {
|
||||
Scope string `json:"s"`
|
||||
TS time.Time `json:"t"`
|
||||
ID string `json:"i"`
|
||||
}
|
||||
|
||||
func EncodeCursor(c Cursor) string {
|
||||
b, _ := json.Marshal(c)
|
||||
return base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
func DecodeCursor(s string) (Cursor, error) {
|
||||
var c Cursor
|
||||
b, err := base64.RawURLEncoding.DecodeString(s)
|
||||
if err != nil {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
if c.Scope == "" || c.ID == "" || c.TS.IsZero() {
|
||||
return c, ErrInvalidCursor
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"mtusi-messenger/internal/model"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrUsernameTaken = errors.New("username taken")
|
||||
ErrUnknownMember = errors.New("unknown member")
|
||||
ErrInvalidCursor = errors.New("invalid cursor")
|
||||
)
|
||||
|
||||
type Store struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
|
||||
|
||||
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
|
||||
|
||||
func IsUnavailable(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) {
|
||||
return true
|
||||
}
|
||||
var connErr *pgconn.ConnectError
|
||||
if errors.As(err, &connErr) {
|
||||
return true
|
||||
}
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
return true
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
|
||||
switch {
|
||||
case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03",
|
||||
len(pgErr.Code) == 5 && pgErr.Code[:2] == "08":
|
||||
return true
|
||||
}
|
||||
}
|
||||
return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed)
|
||||
}
|
||||
|
||||
func pgCode(err error) string {
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) {
|
||||
return pgErr.Code
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) }
|
||||
|
||||
func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) {
|
||||
u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName}
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ($1, $2, $3, $4) RETURNING created_at`,
|
||||
u.ID, username, displayName, passwordHash).Scan(&created)
|
||||
if pgCode(err) == "23505" {
|
||||
return nil, ErrUsernameTaken
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) {
|
||||
u := &model.User{}
|
||||
var created time.Time
|
||||
var hash string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`,
|
||||
username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, hash, nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) {
|
||||
var expires time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`INSERT INTO sessions (token_hash, user_id, expires_at)
|
||||
VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3))
|
||||
RETURNING expires_at`,
|
||||
tokenHash, userID, ttl.Seconds()).Scan(&expires)
|
||||
if err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
return expires.UTC(), nil
|
||||
}
|
||||
|
||||
func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) {
|
||||
u := &model.User{}
|
||||
var created, expires time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`,
|
||||
tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, time.Time{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
u.CreatedAt = utc(created)
|
||||
return u, expires.UTC(), nil
|
||||
}
|
||||
|
||||
func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`UPDATE sessions SET revoked_at = now()
|
||||
WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM sessions
|
||||
WHERE expires_at < now() - make_interval(secs => $1)
|
||||
OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) {
|
||||
c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)}
|
||||
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
|
||||
var created time.Time
|
||||
if err := tx.QueryRow(ctx,
|
||||
`INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`,
|
||||
c.ID, title, creatorID).Scan(&created); err != nil {
|
||||
return err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
_, err := tx.Exec(ctx,
|
||||
`INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT $1, m.user_id, m.ord - 1
|
||||
FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`,
|
||||
c.ID, memberIDs)
|
||||
return err
|
||||
})
|
||||
if pgCode(err) == "23503" {
|
||||
return nil, ErrUnknownMember
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
|
||||
var ok bool
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
|
||||
chatID, userID).Scan(&ok)
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) {
|
||||
c := &model.Chat{}
|
||||
var created time.Time
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
array_agg(m.user_id::text ORDER BY m.position)
|
||||
FROM chats c JOIN chat_members m ON m.chat_id = c.id
|
||||
WHERE c.id = $1
|
||||
AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2)
|
||||
GROUP BY c.id`,
|
||||
chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) {
|
||||
var ts *time.Time
|
||||
var cid *string
|
||||
if cur != nil {
|
||||
ts, cid = &cur.TS, &cur.ID
|
||||
}
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT c.id, c.title, c.created_at,
|
||||
(SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id)
|
||||
FROM chat_members me JOIN chats c ON c.id = me.chat_id
|
||||
WHERE me.user_id = $1
|
||||
AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid))
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4`,
|
||||
userID, ts, cid, limit+1)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]*model.Chat, 0, limit+1)
|
||||
for rows.Next() {
|
||||
c := &model.Chat{}
|
||||
var created time.Time
|
||||
if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
c.CreatedAt = utc(created)
|
||||
out = append(out, c)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(out) > limit
|
||||
if more {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, more, nil
|
||||
}
|
||||
|
||||
var (
|
||||
ErrIdempotencyConflict = errors.New("idempotency key reused with different body")
|
||||
ErrAttachmentNotFound = errors.New("attachment not found in chat")
|
||||
)
|
||||
|
||||
// AddMessage inserts the message, its idempotency record and attachment links in
|
||||
// one statement. Attachments must be ready and belong to the same chat; the
|
||||
// composite FK on message_attachments enforces the chat boundary in the DB too.
|
||||
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) {
|
||||
for attempt := 0; attempt < 2; attempt++ {
|
||||
newID := uuid.NewString()
|
||||
var member, attachmentsOK bool
|
||||
var created *time.Time
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
WITH member AS (
|
||||
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1
|
||||
), att AS (
|
||||
SELECT count(*) = cardinality($8::uuid[]) AS ok
|
||||
FROM attachments
|
||||
WHERE chat_id = $2 AND id = ANY($8::uuid[]) AND status = 'ready'
|
||||
), k AS (
|
||||
INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id)
|
||||
SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok
|
||||
ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now()
|
||||
WHERE message_idempotency.created_at < now() - make_interval(secs => $7)
|
||||
RETURNING message_id
|
||||
), m AS (
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
SELECT $5, $2, $1, $6 FROM k
|
||||
RETURNING created_at
|
||||
), ma AS (
|
||||
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
SELECT $5, $2, a.id, a.ord - 1
|
||||
FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord)
|
||||
WHERE EXISTS (SELECT 1 FROM m)
|
||||
)
|
||||
SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT created_at FROM m)`,
|
||||
senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &created)
|
||||
if pgCode(err) == "23503" {
|
||||
return nil, false, ErrAttachmentNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !member {
|
||||
return nil, false, ErrNotFound
|
||||
}
|
||||
if !attachmentsOK {
|
||||
return nil, false, ErrAttachmentNotFound
|
||||
}
|
||||
if created != nil {
|
||||
return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text,
|
||||
CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil
|
||||
}
|
||||
var stored []byte
|
||||
var at time.Time
|
||||
m := &model.Message{ChatID: chatID}
|
||||
err = s.pool.QueryRow(ctx, `
|
||||
SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at,
|
||||
`+attachmentIDsSQL+`
|
||||
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
|
||||
WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3
|
||||
AND i.created_at >= now() - make_interval(secs => $4)`,
|
||||
senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if !bytes.Equal(stored, reqHash) {
|
||||
return nil, false, ErrIdempotencyConflict
|
||||
}
|
||||
m.CreatedAt = utc(at)
|
||||
return m, true, nil
|
||||
}
|
||||
return nil, false, errors.New("idempotency record kept changing")
|
||||
}
|
||||
|
||||
const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position)
|
||||
FROM message_attachments ma WHERE ma.message_id = m.id), '{}')`
|
||||
|
||||
func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx,
|
||||
`DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) {
|
||||
var (
|
||||
rows pgx.Rows
|
||||
err error
|
||||
)
|
||||
if cur == nil {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1
|
||||
ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1)
|
||||
} else {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
|
||||
FROM messages m
|
||||
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid)
|
||||
ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := make([]*model.Message, 0, limit+1)
|
||||
for rows.Next() {
|
||||
m := &model.Message{}
|
||||
var created time.Time
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
m.CreatedAt = utc(created)
|
||||
out = append(out, m)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
more := len(out) > limit
|
||||
if more {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, more, nil
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 2.5 KiB |
@@ -0,0 +1,34 @@
|
||||
"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git)."""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
body = kind + data
|
||||
return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff)
|
||||
|
||||
|
||||
def png(width, height, raw=None, level=6, color_type=2):
|
||||
"""RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise)."""
|
||||
if raw is None:
|
||||
row = 1 + 3 * width
|
||||
raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height))
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) +
|
||||
chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def near_limit_png(limit=10 * 1024 * 1024):
|
||||
"""Valid noise PNG a little below the byte limit (stored deflate blocks)."""
|
||||
width = 1000
|
||||
height = (limit - 64 * 1024) // (1 + 3 * width)
|
||||
data = png(width, height, level=0)
|
||||
assert len(data) <= limit, len(data)
|
||||
return data
|
||||
|
||||
|
||||
def bomb_png(width=20000, height=20000):
|
||||
"""Header declares 400M pixels, the compressed payload is tiny."""
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) +
|
||||
chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b''))
|
||||
@@ -0,0 +1,175 @@
|
||||
#!/usr/bin/env python3
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import test_attachments
|
||||
|
||||
RESULTS = []
|
||||
|
||||
|
||||
def case(name):
|
||||
def wrap(fn):
|
||||
def run(*a):
|
||||
try:
|
||||
detail = fn(*a)
|
||||
RESULTS.append({'test': name, 'result': 'PASS', 'detail': detail})
|
||||
print('PASS', name, '-', detail or '')
|
||||
except Failure as exc:
|
||||
RESULTS.append({'test': name, 'result': 'FAIL', 'detail': str(exc)})
|
||||
print('FAIL', name, '-', exc)
|
||||
return run
|
||||
return wrap
|
||||
|
||||
|
||||
def msg_path(chat):
|
||||
return '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
out, cursor = [], None
|
||||
while True:
|
||||
q = '?limit=100' + ('&cursor=' + cursor if cursor else '')
|
||||
page = c.call('GET', msg_path(chat) + q, actor=actor, schema='MessagePage')['json']
|
||||
out += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if not cursor:
|
||||
return out
|
||||
|
||||
|
||||
@case('one session served by >=2 replicas without re-login (no stickiness)')
|
||||
def t_no_sticky(c, alice, bob, chat):
|
||||
seen = {}
|
||||
for _ in range(30):
|
||||
iid = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
seen[iid] = seen.get(iid, 0) + 1
|
||||
check(len(seen) >= 2, 'only one replica observed: %s' % seen)
|
||||
return 'distribution %s' % sorted(seen.values())
|
||||
|
||||
|
||||
@case('X-Instance-Id is present on errors and health too')
|
||||
def t_instance_on_errors(c, alice, bob, chat):
|
||||
r = c.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice)
|
||||
check(bool(r['headers'].get('X-Instance-Id')), 'missing on 400')
|
||||
r = c.call('GET', '/api/v1/users/me', 401)
|
||||
check(bool(r['headers'].get('X-Instance-Id')), 'missing on 401')
|
||||
r = c.call('GET', '/health/ready', schema='Health')
|
||||
check(bool(r['headers'].get('X-Instance-Id')), 'missing on health')
|
||||
|
||||
|
||||
@case('Idempotency-Key missing or malformed -> 400, nothing stored')
|
||||
def t_key_validation(c, alice, bob, chat):
|
||||
before = len(history(c, alice, chat))
|
||||
c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'без ключа'})
|
||||
for bad in ('', 'a b', 'clé', 'x' * 129, 'a/b'):
|
||||
c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'плохой ключ'},
|
||||
headers={'Idempotency-Key': bad})
|
||||
c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'граница ключа'},
|
||||
headers={'Idempotency-Key': 'A-z0_9.:' + 'k' * 120})
|
||||
check(len(history(c, alice, chat)) == before + 1, 'rejected requests created messages')
|
||||
|
||||
|
||||
@case('concurrent replay of one key through different replicas -> exactly one message')
|
||||
def t_concurrent_same_key(c, alice, bob, chat):
|
||||
key = 'conc-' + uuid.uuid4().hex
|
||||
text = 'Конкурентный повтор ' + key
|
||||
n = 24
|
||||
barrier = threading.Barrier(n)
|
||||
raw = json.dumps({'text': text}, ensure_ascii=False).encode()
|
||||
|
||||
def send(_):
|
||||
local = Client(c.base, timeout=10)
|
||||
barrier.wait()
|
||||
r = local.call('POST', msg_path(chat), 201, actor=alice, raw=raw,
|
||||
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
|
||||
return r['json'], r['headers']['X-Instance-Id']
|
||||
|
||||
with ThreadPoolExecutor(max_workers=n) as pool:
|
||||
results = list(pool.map(send, range(n)))
|
||||
bodies = {json.dumps(m, sort_keys=True) for m, _ in results}
|
||||
instances = {i for _, i in results}
|
||||
check(len(bodies) == 1, 'replays returned different messages: %d' % len(bodies))
|
||||
check(len(instances) >= 2, 'all requests hit one replica; test does not prove cross-replica')
|
||||
same = [m for m in history(c, bob, chat) if m['text'] == text]
|
||||
check(len(same) == 1, 'expected exactly 1 stored message, found %d' % len(same))
|
||||
return '%d parallel requests via %d replicas -> 1 message' % (n, len(instances))
|
||||
|
||||
|
||||
@case('replay is semantic: JSON key order/whitespace does not matter; other body -> 409')
|
||||
def t_semantic_replay(c, alice, bob, chat):
|
||||
key = uuid.uuid4().hex
|
||||
first = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{"text":"abc"}',
|
||||
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
|
||||
again = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{ "text" : "abc" }\n',
|
||||
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
|
||||
check(first['json'] == again['json'], 'replay returned another message')
|
||||
c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'abd'}, headers={'Idempotency-Key': key})
|
||||
third = c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'abc'},
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
check(third['json'] == first['json'], 'stored result changed after 409')
|
||||
|
||||
|
||||
@case('key scope = user + chat: same key is independent for other user / other chat')
|
||||
def t_key_scope(c, alice, bob, chat):
|
||||
key = 'shared-' + uuid.uuid4().hex
|
||||
a = c.message(alice, chat, 'scope', key=key)['json']
|
||||
b = c.message(bob, chat, 'scope', key=key)['json']
|
||||
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Второй', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
|
||||
a2 = c.message(alice, chat2, 'scope', key=key)['json']
|
||||
check(len({a['id'], b['id'], a2['id']}) == 3, 'scopes collided')
|
||||
check(b['sender_id'] == bob['id'], 'sender mixed up')
|
||||
|
||||
|
||||
@case('replay re-checks the session and chat access')
|
||||
def t_replay_rechecks_access(c, alice, bob, chat, eve):
|
||||
key = uuid.uuid4().hex
|
||||
c.message(alice, chat, 'access', key=key)
|
||||
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'access'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', msg_path(chat), 401, body={'text': 'access'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': ' '}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'x'})
|
||||
|
||||
|
||||
@case('logout is visible on every replica immediately (shared sessions)')
|
||||
def t_logout_shared(c, alice, bob, chat):
|
||||
tmp = c.user('tmp')
|
||||
c.call('GET', '/api/v1/users/me', actor=tmp, schema='User')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=tmp)
|
||||
seen = set()
|
||||
for _ in range(10):
|
||||
seen.add(c.call('GET', '/api/v1/users/me', 401, actor=tmp)['headers']['X-Instance-Id'])
|
||||
check(len(seen) >= 2, 'revocation checked on one replica only')
|
||||
return 'revoked token rejected by %d replicas' % len(seen)
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--base-url', default='http://localhost:8080')
|
||||
p.add_argument('--json', default=None)
|
||||
a = p.parse_args()
|
||||
c = Client(a.base_url, timeout=60)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
for t in (t_no_sticky, t_instance_on_errors, t_key_validation, t_concurrent_same_key,
|
||||
t_semantic_replay, t_key_scope, t_logout_shared):
|
||||
t(c, alice, bob, chat)
|
||||
t_replay_rechecks_access(c, alice, bob, chat, eve)
|
||||
for needs_eve, t in test_attachments.register(case, history, msg_path):
|
||||
t(c, alice, bob, chat, *([eve] if needs_eve else []))
|
||||
failed = [r for r in RESULTS if r['result'] != 'PASS']
|
||||
print('%d/%d passed' % (len(RESULTS) - len(failed), len(RESULTS)))
|
||||
if a.json:
|
||||
Path(a.json).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,181 @@
|
||||
"""Lab4 attachment cases; registered by run_tests.py (same case() harness)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
|
||||
from client import Client, check
|
||||
import imagegen
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / 'fixtures'
|
||||
PNG = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||||
JPG = (FIXTURES / 'sample.jpg').read_bytes()
|
||||
|
||||
|
||||
def register(case, history, msg_path):
|
||||
"""Return the list of (needs_eve, fn) cases bound to the runner's helpers."""
|
||||
|
||||
def upload(c, actor, chat, data, ctype='image/png', name='sample.png', expected=201):
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(name, safe=''),
|
||||
expected, actor=actor, raw=data, headers={'Content-Type': ctype},
|
||||
schema='Attachment' if expected == 201 else None)
|
||||
return r['json']
|
||||
|
||||
@case('PNG and JPEG upload/download byte-exact with true Content-Type')
|
||||
def t_upload_formats(c, alice, bob, chat):
|
||||
for data, ctype, name in ((PNG, 'image/png', 'a.png'), (JPG, 'image/jpeg', 'b.jpg')):
|
||||
att = upload(c, alice, chat, data, ctype, name)
|
||||
check(att['content_type'] == ctype and att['size_bytes'] == len(data), 'metadata mismatch')
|
||||
check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'sha256 mismatch')
|
||||
check(att['status'] == 'ready' and att['variants'] == [] and att['error_code'] is None, 'lab4 state')
|
||||
meta = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
|
||||
check(meta == att, 'GET metadata differs from POST')
|
||||
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
|
||||
check(got['bytes'] == data and got['headers'].get_content_type() == ctype, 'download differs')
|
||||
check(got['headers'].get('X-Content-Type-Options') == 'nosniff', 'nosniff missing')
|
||||
return 'png %d B, jpeg %d B' % (len(PNG), len(JPG))
|
||||
|
||||
@case('size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413')
|
||||
def t_size_limit(c, alice, bob, chat):
|
||||
near = imagegen.near_limit_png()
|
||||
att = upload(c, alice, chat, near, name='near-limit.png')
|
||||
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=alice)
|
||||
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'] == hashlib.sha256(near).hexdigest(),
|
||||
'big file changed')
|
||||
over = PNG + b'\0' * (10 * 1024 * 1024 + 1 - len(PNG))
|
||||
upload(c, alice, chat, over, name='over.png', expected=413)
|
||||
return 'accepted %d B, rejected %d B' % (len(near), len(over))
|
||||
|
||||
@case('decompression bomb (400M px header, tiny body) -> 413 before decoding')
|
||||
def t_bomb(c, alice, bob, chat):
|
||||
data = imagegen.bomb_png()
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bomb.png', 413,
|
||||
actor=alice, raw=data, headers={'Content-Type': 'image/png'})
|
||||
check(r['elapsed'] < 2, 'bomb took %.2fs: was it decoded?' % r['elapsed'])
|
||||
return '%d B rejected in %.0f ms' % (len(data), r['elapsed'] * 1000)
|
||||
|
||||
@case('corrupted images and MIME spoofing -> 415, never 500')
|
||||
def t_corrupt_and_spoof(c, alice, bob, chat):
|
||||
cases = {
|
||||
'truncated png': (PNG[:len(PNG) // 2], 'image/png'),
|
||||
'garbage after png header': (PNG[:40] + os.urandom(2000), 'image/png'),
|
||||
'truncated jpeg': (JPG[:len(JPG) // 2], 'image/jpeg'),
|
||||
'png declared as jpeg': (PNG, 'image/jpeg'),
|
||||
'jpeg declared as png': (JPG, 'image/png'),
|
||||
'gif bytes as png': (b'GIF89a' + os.urandom(100), 'image/png'),
|
||||
'html as png': (b'<html><script>alert(1)</script></html>', 'image/png'),
|
||||
'text/plain content type': (PNG, 'text/plain'),
|
||||
'multipart content type': (PNG, 'multipart/form-data; boundary=x'),
|
||||
}
|
||||
for name, (data, ctype) in cases.items():
|
||||
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=x.png', 415,
|
||||
actor=alice, raw=data, headers={'Content-Type': ctype})
|
||||
check(r['json']['error']['code'] == 'unsupported_media_type', name)
|
||||
upload(c, alice, chat, b'', name='empty.png', expected=400)
|
||||
return '%d spoof/corrupt variants -> 415' % len(cases)
|
||||
|
||||
@case('dangerous filenames -> 400; Unicode display name kept as-is')
|
||||
def t_filenames(c, alice, bob, chat):
|
||||
bad = ['../etc/passwd', 'a/b.png', 'a\\b.png', 'nul\x00.png', 'bell\x07.png', 'line\nbreak.png',
|
||||
'', ' ', '.', '..', 'x' * 256, 'evilgnp.exe']
|
||||
for name in bad:
|
||||
upload(c, alice, chat, PNG, name=name, expected=400)
|
||||
base = '/api/v1/chats/' + chat['id'] + '/attachments'
|
||||
c.call('POST', base, 400, actor=alice, raw=PNG, headers={'Content-Type': 'image/png'})
|
||||
c.call('POST', base + '?filename=a.png&filename=b.png', 400, actor=alice, raw=PNG,
|
||||
headers={'Content-Type': 'image/png'})
|
||||
nice = 'Фото кота 🐱 (1).png'
|
||||
att = upload(c, alice, chat, PNG, name=nice)
|
||||
check(att['filename'] == nice, 'filename changed')
|
||||
long_ok = upload(c, alice, chat, PNG, name='я' * 255)
|
||||
check(len(long_ok['filename']) == 255, '255 code points must pass')
|
||||
return '%d bad names rejected' % len(bad)
|
||||
|
||||
@case('ACL: outsider gets 404 on upload, metadata, content; no token -> 401')
|
||||
def t_attachment_acl(c, alice, bob, chat, eve):
|
||||
att = upload(c, alice, chat, PNG)
|
||||
upload(c, eve, chat, PNG, expected=404)
|
||||
for suffix in ('', '/content', '/content?variant=original'):
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401)
|
||||
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), 404, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/not-a-uuid', 400, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=huge', 400, actor=alice)
|
||||
|
||||
@case('message attachment_ids: other chat / foreign / unknown -> 404, nothing stored')
|
||||
def t_message_links(c, alice, bob, chat, eve):
|
||||
# alice is a member of both chats: the file still must not cross the chat boundary
|
||||
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
|
||||
other = upload(c, alice, chat2, PNG)
|
||||
eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve,
|
||||
body={'title': 'Чат Евы', 'member_ids': [eve['id']]}, schema='Chat')['json']
|
||||
foreign = upload(c, eve, eve_chat, PNG)
|
||||
own = upload(c, alice, chat, PNG)
|
||||
before = len(history(c, alice, chat))
|
||||
for ids in ([other['id']], [foreign['id']], [str(uuid.uuid4())], [own['id'], other['id']]):
|
||||
r = c.call('POST', msg_path(chat), 404, actor=alice, body={'text': 'x', 'attachment_ids': ids},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
check(r['json']['error']['code'] == 'not_found', 'expected not_found')
|
||||
check(len(history(c, alice, chat)) == before, 'rejected message was stored')
|
||||
m = c.message(bob, chat, '', attachments=[own['id']])['json']
|
||||
check(m['attachment_ids'] == [own['id']] and m['text'] == '', 'bob could not reference chat file')
|
||||
listed = [x for x in history(c, alice, chat) if x['id'] == m['id']]
|
||||
check(listed == [m], 'history lost attachment_ids')
|
||||
|
||||
@case('attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400')
|
||||
def t_message_validation(c, alice, bob, chat):
|
||||
att = upload(c, alice, chat, PNG)
|
||||
for body in ({'text': 'x', 'attachment_ids': [att['id'], att['id']]},
|
||||
{'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]},
|
||||
{'text': 'x', 'attachment_ids': ['nope']},
|
||||
{'text': 'x', 'attachment_ids': 'nope'},
|
||||
{'text': '', 'attachment_ids': []},
|
||||
{'text': ' '}):
|
||||
c.call('POST', msg_path(chat), 400, actor=alice, body=body,
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
ten = [upload(c, alice, chat, PNG)['id'] for _ in range(10)]
|
||||
m = c.message(alice, chat, 'десять файлов', attachments=ten)['json']
|
||||
check(m['attachment_ids'] == ten, 'order of attachment_ids changed')
|
||||
|
||||
@case('idempotency: missing attachment_ids == []; other attachment list -> 409')
|
||||
def t_attachment_idempotency(c, alice, bob, chat):
|
||||
key = uuid.uuid4().hex
|
||||
first = c.message(alice, chat, 'idem', key=key)['json']
|
||||
again = c.message(alice, chat, 'idem', key=key, attachments=[])['json']
|
||||
check(first == again and first['attachment_ids'] == [], 'missing vs [] not equal')
|
||||
att = upload(c, alice, chat, PNG)
|
||||
c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'idem', 'attachment_ids': [att['id']]},
|
||||
headers={'Idempotency-Key': key})
|
||||
key2 = uuid.uuid4().hex
|
||||
a = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json']
|
||||
b = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json']
|
||||
check(a == b, 'replay with attachments differs')
|
||||
|
||||
@case('10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back')
|
||||
def t_concurrent_uploads(c, alice, bob, chat):
|
||||
n = 10
|
||||
barrier = threading.Barrier(n)
|
||||
|
||||
def one(i):
|
||||
local = Client(c.base, timeout=60)
|
||||
barrier.wait()
|
||||
r = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=p%d.png' % i, 201,
|
||||
actor=alice, raw=PNG, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
return r['json'], r['headers']['X-Instance-Id']
|
||||
|
||||
with ThreadPoolExecutor(max_workers=n) as pool:
|
||||
res = list(pool.map(one, range(n)))
|
||||
check(len({a['id'] for a, _ in res}) == n, 'ids collided')
|
||||
for a, _ in res:
|
||||
got = c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=bob)
|
||||
check(got['bytes'] == PNG, 'content mixed up')
|
||||
return '%d uploads via %d replicas' % (n, len({i for _, i in res}))
|
||||
|
||||
return [(False, t_upload_formats), (False, t_size_limit), (False, t_bomb), (False, t_corrupt_and_spoof),
|
||||
(False, t_filenames), (False, t_message_validation), (False, t_attachment_idempotency),
|
||||
(False, t_concurrent_uploads), (True, t_attachment_acl), (True, t_message_links)]
|
||||
Reference in New Issue
Block a user