Author SHA1 Message Date
Sh1pYe e094e13dc3 Fixed 2026-10-29 19:51:04 +03:00
Sh1pYe 1706ddca25 Fixed 2026-10-29 15:00:36 +03:00
56 changed files with 5935 additions and 19 deletions
+15 -2
View File
@@ -7,14 +7,27 @@ SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_ENDPOINT=http://s3:8333
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Student implementation knobs (see REPORT.md)
SESSION_CACHE_MAX_TTL_SECONDS=60
DB_MAX_CONNS=10
API_CPUS=1.0
API_MEMORY=512M
API_GOMEMLIMIT=400MiB
SHUTDOWN_DRAIN_MS=3000
S3_TIMEOUT=15s
UPLOAD_CONCURRENCY=8
DECODE_BUDGET_BYTES=134217728
UPLOAD_TIMEOUT=60s
RECONCILE_GRACE=15m
RECONCILE_EVERY=10m
+25
View File
@@ -0,0 +1,25 @@
# syntax=docker/dockerfile:1
FROM golang:1.23.4-alpine3.21 AS build
WORKDIR /src
COPY src/go.mod src/go.sum ./
RUN go mod download
COPY src/ ./
ENV CGO_ENABLED=0
RUN go build -trimpath -ldflags="-s -w" -o /out/server ./cmd/server
FROM alpine:3.21.2 AS runtime
RUN addgroup -S app && adduser -S -G app app
WORKDIR /app
COPY --from=build /out/server /app/server
ARG APP_BUILD=dev
ENV APP_BUILD=$APP_BUILD
LABEL org.opencontainers.image.revision=$APP_BUILD
USER app
EXPOSE 8080
ENV APP_HOST=0.0.0.0 APP_PORT=8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null http://127.0.0.1:8080/health/ready || exit 1
ENTRYPOINT ["/app/server"]
+37
View File
@@ -28,3 +28,40 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---- student targets (lab4 solution) ----
.PHONY: secrets up down ps restore test-student ops-checks memory track2-faults track2-cost
secrets:
$(PYTHON) scripts/init_secrets.py
up: secrets
docker compose up -d --build --wait
down:
docker compose stop
ps:
docker compose ps -a
restore:
docker compose up -d --wait
test-student:
$(PYTHON) student-tests/run_tests.py --base-url "$(BASE_URL)" --json evidence/student-tests.json
ops-checks:
$(PYTHON) scripts/lab4ops.py private --output evidence/private.json
$(PYTHON) scripts/lab4ops.py failover-upload --output evidence/failover-upload.json
$(PYTHON) scripts/lab4ops.py no-local-file --output evidence/no-local-file.json
$(PYTHON) scripts/lab4ops.py bootstrap --output evidence/bootstrap.json
$(PYTHON) scripts/lab4ops.py s3-outage --output evidence/s3-outage.json
memory:
$(PYTHON) scripts/lab4ops.py memory --output evidence/memory.json
track2-faults:
$(PYTHON) scripts/track2/run_track2.py faults --output evidence/track2/faults.json
track2-cost:
$(PYTHON) scripts/track2/run_track2.py cost --output evidence/track2/cost.json
+163 -17
View File
@@ -1,37 +1,183 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Кораблев Петр, БВТ2304
- Целевая оценка: **5** (уровень 3 + уровень 4 + **трек 2 «Согласованность двух хранилищ»**)
- Commit/PR: ветка `solution/BVT2304-korablev`
- Использованные источники, библиотеки и помощь ИИ: Go 1.23.4, net/http, pgx/v5 5.7.2, go-redis/v9 9.7.0, golang.org/x/crypto (Argon2id), golang.org/x/sync 0.10.0 (semaphore), google/uuid 1.6.0, **minio-go/v7 7.0.82** (S3-клиент, работает с любым S3-совместимым сервером), стандартные `image/png` и `image/jpeg`. Образы: PostgreSQL 17.2-alpine3.21, Valkey 8.0.2-alpine3.21, HAProxy 3.0.5-alpine, **SeaweedFS 3.80** (`chrislusf/seaweedfs:3.80`), golang:1.23.4-alpine3.21 → alpine:3.21.2; для проверок вне API — python:3.12.8-alpine3.21. Документация: AWS S3 (uploads, SigV4, ListObjectsV2), SeaweedFS wiki (S3 API, `s3.config` identities), Go `image.DecodeConfig`. Помощь ИИ использовалась при написании отчёта.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker 29.x + Compose v2 (Docker Desktop/WSL2), Python 3.9+, Git. Go и make локально не нужны (сборка в Docker; без make команды ниже запускаются через `python`). На Windows тесты запускать с `PYTHONUTF8=1` (контракт в UTF-8).
```sh
python3 scripts/init_secrets.py # = make secrets: secrets/* + .env из .env.example
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait # = make up
make check && make test # smoke через LB http://localhost:8080
make test-student # собственные тесты (18 кейсов)
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
docker compose up -d --no-recreate --wait api # вернуть остановленную реплику
make ops-checks memory track2-faults track2-cost # уровень 4 и трек 2 (нужен Docker)
```
- `init_secrets.py` генерирует случайные пароли PostgreSQL/Valkey и **две S3-идентичности**: `bootstrap-admin` (только для `s3-init`) и `messenger-api` (права `Read/Write/List` только на bucket `messenger-attachments`), а также `secrets/s3_identities.json` — конфиг SeaweedFS. Всё в `secrets/` (в `.gitignore` и `.dockerignore`), контейнерам передаётся как Compose secrets `/run/secrets/*`.
- Порядок старта задан `depends_on`: postgres/valkey/s3 healthy → `migrate` (миграции под advisory lock) → `s3-init` (идемпотентное создание bucket) → `api` ×2 → `lb`; `reconciler` стартует после `s3-init`. Готовность — `--wait`/`/health/ready`.
- Безопасная остановка: `docker compose stop` (`make down`), данные в volumes `pgdata`, `s3data` сохраняются. Намеренное удаление учебных данных: `docker compose down -v` (удаляет volumes — необратимо).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
client → HAProxy (lb, 127.0.0.1:8080) → api ×2 (Go, read-only rootfs)
├─ PostgreSQL: users, chats, messages, sessions, idempotency,
│ attachments (метаданные), message_attachments
├─ Valkey: кэш сессий (из lab3)
└─ SeaweedFS S3 (bucket messenger-attachments, приватный): байты
reconciler (тот же образ, `reconcile --apply --every 10m`) — сверка БД ↔ S3
```
**Перенесено из lab3** без изменения поведения: пользователи, Basic → Bearer-сессии (Argon2id, Valkey-кэш), чаты, keyset-пагинация, обязательный `Idempotency-Key`, `X-Instance-Id`, HAProxy с health-check, хуки restart/stop-one. **Изменено:** убран экспериментальный путь `MESSAGE_WRITE_PATH=tx` трека 1 lab3 (остался один путь — one-shot CTE), лимит памяти API 256M → 512M, таймауты HAProxy client/server 30s/10s → 75s (загрузка до 10 MiB).
**Выбор S3-сервера.** Планировался MinIO, но на 2026-10 образы `minio/minio` и `minio/mc` недоступны ни в Docker Hub, ни в quay.io (pull access denied / 401). Выбран SeaweedFS 3.80: одна команда `weed server -s3` (master+volume+filer+S3), идентичности с правами на конкретный bucket через `-s3.config`, без анонимной идентичности — значит bucket приватный. S3-порт 8333 не публикуется, консоли master (9333) и filer (8888) тоже: сервис только в сети `data`.
**Модель данных** (`src/internal/db/migrations/0003_attachments.sql`):
- `attachments`: `id`, `chat_id`, `owner_id`, `filename` (только для отображения), `content_type`, `size_bytes`, `sha256`, `width`, `height`, `object_key` (UNIQUE, генерирует сервер: `attachments/<chat_id>/<attachment_id>`), `status ∈ {pending, ready}`, `missing_at`, `created_at`, `updated_at`. FK `(chat_id, owner_id) → chat_members`, CHECK-и на размер ≤ 10 MiB, пиксели ≤ 40M, формат sha256.
- `message_attachments(message_id, chat_id, attachment_id, position)` с составным FK `(chat_id, attachment_id) → attachments(chat_id, id)`: **сама БД** не даёт привязать к сообщению файл другого чата.
- `messages.text` теперь может быть пустым (CHECK 0..4000); правило «пустой текст только при ≥1 вложении» проверяет API.
**Загрузка** `POST /api/v1/chats/{id}/attachments?filename=…` (`src/internal/api/attachments.go`):
1. Аутентификация → членство в чате (посторонний получает 404 до чтения тела) → `filename` (1..255 code points, без `/`, `\`, NUL, управляющих и bidi-override символов, не `.`/`..`) → `Content-Type` ∈ {image/png, image/jpeg} иначе 415 → `Content-Length > 10 MiB` → 413 сразу.
2. Слот загрузки (семафор `UPLOAD_CONCURRENCY=8` на реплику), тело читается через `http.MaxBytesReader(10 MiB)` → 413 при превышении. Тело буферизуется в памяти (≤ 10 MiB на слот), значит память одной реплики под загрузки ограничена 8 × 10 MiB.
3. Формат определяется по magic bytes и сравнивается с заявленным (подмена → 415); `image.DecodeConfig` читает только заголовок: >40M пикселей → 413 **до** декодирования; затем полное декодирование под общим бюджетом памяти (`DECODE_BUDGET_BYTES=128 MiB`, взвешенный семафор по оценке `пиксели × байт/пиксель`) — повреждённые данные → 415.
4. Транзакционная граница: **(a)** `INSERT attachments status='pending'` → **(b)** S3 `PUT` (Content-MD5) → **(c)** `UPDATE … SET status='ready' WHERE status='pending'` → `201 ready`. 201 возвращается только когда и байты, и метаданные зафиксированы.
5. Ошибка на (b) или (c) → 503 и фоновая компенсация, которая начинается со строки: `DELETE … WHERE status='pending'`, и только если строка удалена — удаление объекта. Если `UPDATE → ready` на самом деле закоммитился, а ответ БД потерялся, строка уже ready, и объект не трогаем (иначе получилось бы ready-вложение без байтов). Если недоступна БД, остаётся stale pending; если недоступен S3, может остаться объект без строки. Оба случая убирает reconciler после grace.
**Скачивание** `GET /attachments/{id}/content`: только участники чата (иначе 404), только `status='ready'`; `Stat` до первого байта ответа (отсутствие объекта → 404, недоступность S3 → 503), затем поток из S3 в ответ, без redirect; заголовки `Content-Type` (истинный), `Content-Length`, `X-Content-Type-Options: nosniff`, `Cache-Control: private, no-store`, `ETag: "<sha256>"`. `variant=thumbnail|crop|optimized` → 404 (вариантов в lab4 нет), неизвестный → 400.
**Сообщение** с `attachment_ids`: 0..10 уникальных UUID (иначе 400). Проверка «все ready и в этом чате», запись idempotency, сообщения и связей — **одним SQL-оператором** (CTE), как в lab3. Чужое, неизвестное или из другого чата → 404, ничего не сохраняется. Отпечаток идемпотентности считается по `{text, attachment_ids}` с нормализацией «нет поля» = `[]`.
**Готовность и отказы.** `/health/ready` зависит только от PostgreSQL. Недоступность S3 ломает только эндпоинты вложений (503), а текстовый мессенджер продолжает работать: если бы ready зависел от S3, LB снял бы обе реплики и упал бы весь API. SDK S3 настроен на быстрый отказ: `MaxRetry=1`, dial-таймаут 2 с, `ResponseHeaderTimeout` 10 с, операции под `S3_TIMEOUT=15s`.
**Соответствие `.env.example`** — все переменные реально используются в `compose.yaml`: `S3_ENDPOINT=http://s3:8333`, `S3_REGION`, `S3_BUCKET`, `S3_*_FILE` (secrets), `MAX_UPLOAD_BYTES` (не больше 10 MiB, иначе старт с ошибкой), `MAX_IMAGE_PIXELS`, плюс `UPLOAD_CONCURRENCY`, `DECODE_BUDGET_BYTES`, `UPLOAD_TIMEOUT`, `S3_TIMEOUT`, `API_MEMORY`, `API_GOMEMLIMIT`, `RECONCILE_GRACE`, `RECONCILE_EVERY`.
| Роль | Service | Образ | Health | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| LB | lb | haproxy:3.0.5-alpine | `:8404/healthz` | — | app; 127.0.0.1:8080 | — |
| API ×2 | api | mtusi-messenger:lab4 (сборка) | `/health/ready` | нет (read-only rootfs, tmpfs /tmp 1m) | app, data | secrets: database_url, cache_url, s3_access_key_id, s3_secret_access_key |
| БД | postgres | postgres:17.2-alpine3.21 | pg_isready | pgdata | data | pg_password |
| Сессии | valkey | valkey/valkey:8.0.2-alpine3.21 | ping | — | data | valkey_auth |
| S3 | s3 | chrislusf/seaweedfs:3.80 | `:8333/healthz` | s3data | data, не публикуется | s3_identities.json |
| Bootstrap | migrate, s3-init | mtusi-messenger:lab4 | one-shot | — | data | database_url / s3_admin_* |
| Сверка | reconciler | mtusi-messenger:lab4 | — | — | data | database_url, s3 api-ключи |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Окружение всех замеров: Windows 11, Docker Desktop 29.7.2 / Compose 5.5.0, WSL2 kernel 6.18; лимиты: API 1 CPU / 512M ×2, S3 1 CPU / 1G, PostgreSQL 2 CPU / 1G, reconciler 0.5 CPU / 128M. Дата — 2026-10-29.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Уровень 3
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | `make check`, `make test` | PASS: lab4 public API scenario | evidence/check-smoke.txt |
| Возможности lab3 сохранены | `make test-student` (8 кейсов lab3) | no-sticky, идемпотентность через 2 реплики, logout на всех репликах — PASS | evidence/student-tests.txt |
| S3 в общем стенде, байты в приватном bucket | `lab4ops.py private` | аноним GET/LIST/CreateBucket → 403; api-ключ GET → 200, sha256 совпадает; порт S3 не опубликован | evidence/private.json |
| Метаданные в PostgreSQL, не байты в БД/на диске API | `lab4ops.py no-local-file` | у API read-only rootfs (`touch` → Read-only file system), монтируются только secrets, новых файлов 0 | evidence/no-local-file.json |
| 201 ready, побайтный download, ACL | smoke + `t_upload_formats`, `t_attachment_acl` | PNG и JPEG: sha256/размер/Content-Type совпадают; посторонний получает 404 на метаданные/content/upload, без токена 401 | evidence/student-tests.txt |
| Реальные PNG/JPEG ≤10 MiB, ≤40M px, MIME по содержимому, опасные filename | `t_size_limit`, `t_bomb`, `t_corrupt_and_spoof`, `t_filenames` | PNG 10 420 335 B → 201; 10 MiB+1 → 413; заголовок 20000×20000 → 413 за 32 мс; 9 вариантов подмены/повреждения → 415; 12 опасных имён → 400 | evidence/student-tests.txt |
| Upload → стоп обслужившей реплики → download через другую | `lab4ops.py failover-upload` | загрузка через `api-2`, `docker stop api-2`, 10/10 скачиваний через `api-1` с верным sha256 | evidence/failover-upload.json |
| Failover (публичный тест) | `make test-failover` для каждой реплики | остановлена api-2: PASS, 123 записи, 0 ошибок, восстановление за 4.1 с; остановлена api-1: PASS, 116 записей, 0 ошибок, 4.1 с; файл цел в обоих случаях | evidence/failover.txt |
| Restart всего стенда с volumes | `make test-persistence` (`restart.cmd`) | PASS; postgres/valkey/s3/api перезапущены (uptime до/после в файле), файл и сессия сохранились | evidence/persistence.txt |
### Уровень 4
| Критерий | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Тесты: большой файл, повреждённое изображение, подмена MIME, чужой attachment_id, файл другого чата; нет 500 | `make test-student` (10 кейсов lab4) | 18/18 PASS; в логах API за все прогоны нет ни одного 500 | evidence/student-tests.txt |
| Нет вечных неучтённых объектов: компенсация + сверка с grace | pending-intent + компенсация + `reconciler` (grace 15 мин, каждые 10 мин) | см. трек 2; реальный случай: после падения S3 reconciler нашёл 19 stale pending из прерванных загрузок и удалил их | evidence/track2/faults.json |
| Отказ S3 | `lab4ops.py s3-outage` | upload 503 за 2.0 с, download 503 за 2.0 с, метаданные 200, текстовое сообщение 201, ready 200; после старта S3 загрузки сразу работают, старый файл цел; pending-строк не осталось (компенсация удалила строку) | evidence/s3-outage.json |
| Ограниченная RAM, ≥10 одновременных загрузок | `make memory` | см. «Эксперименты»: пик API ≤ 311 MiB при 40 одновременных 10-МБ загрузках, плато уже после 20 | evidence/memory.json |
| Bootstrap идемпотентен | `lab4ops.py bootstrap` (двойной запуск `migrate` и `s3-init`) | «migrations up to date», `created:false` | evidence/bootstrap.json |
| Отдельные ограниченные S3 credentials; консоль закрыта; private bucket проверен вне API | `lab4ops.py private` | api-идентичность: CreateBucket → 403, PUT в другой bucket → 403; S3/консоли не опубликованы; аноним → 403 | evidence/private.json |
### Трек 5.2 «Согласованность двух хранилищ»
| Критерий | Команда | Результат | Файл |
| --- | --- | --- | --- |
| Повторяемый reconciler с dry-run | `server reconcile [--apply] [--grace] [--every]` | dry-run по умолчанию; advisory lock — одновременно идёт один проход | src/internal/reconcile/reconcile.go |
| Ошибки на обеих сторонах: обнаружение и исправление | `make track2-faults` | 5 из 5 внедрённых сбоев найдены, 7 исправлений, повторный apply → 0 | evidence/track2/faults.json |
| Свежие и используемые не удаляются | там же | свежий orphan, свежий pending и используемые U1/U2 остались целы | evidence/track2/faults.json |
| Стоимость обхода | `make track2-cost` | линейно, ~70 мкс/объект, 20k за 1.4–1.6 с | evidence/track2/cost.json |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### Память API при одновременных загрузках (уровень 4)
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
**Гипотеза:** пиковая память реплики ограничена `≈ база + UPLOAD_CONCURRENCY × 10 MiB + бюджет декодирования`, поэтому она выходит на плато и не растёт с числом клиентов.
**Методика:** `scripts/lab4ops.py memory`. Синтетический шумовой PNG 1000×3480, 10 420 335 B (почти лимит). Перед каждым прогоном `docker compose restart api`, одна прогревочная загрузка, затем N одновременных загрузок (барьер) через LB. Пик читается из cgroup v2 `memory.peak` каждой реплики. По 2 повтора.
| Одновременно | Успешно | Wall, с | p50 / max, мс | Пик API, MiB (max из 2 реплик) |
| --- | --- | --- | --- | --- |
| 1 | 1/1, 1/1 | 0.35 | 337–344 | 64.4 / 66.4 |
| 10 | 10/10, 10/10 | 3.1–4.1 | 2844–3830 / 2882–3945 | 148.2 / 151.7 |
| 20 | 20/20, 20/20 | 6.4–7.1 | 5476–6097 / 6062–6830 | 286.0 / 263.7 |
| 40 | 40/40, 40/40 | 11.8–12.6 | 8542–9590 / 11156–12044 | 264.5 / 311.1 |
**Вывод:** память выходит на плато ~250–310 MiB между 20 и 40 клиентами (слоты по 8 на реплику). Лишние клиенты ждут слота, это видно по росту latency, а не памяти. Это ниже лимита 512M и GOMEMLIMIT 400MiB. **Ограничения:** буфер одного тела ≤10 MiB в RAM — сознательное упрощение вместо потокового парсера PNG. 40M-пиксельный RGBA PNG при декодировании занимает ~160 MB и упирается в бюджет 128 MiB (декодируется один, остальные ждут). Запуск на одном ноутбуке, клиент и стенд делят CPU.
**Найденный отказ:** первый прогон шёл с лимитом S3 512M, и на 20 параллельных PUT процесс SeaweedFS перезапустился (RestartCount 1, логов завершения нет — kill по памяти). 18 из 20 загрузок получили корректный 503, их pending-строки остались (тогдашняя версия компенсации сначала удаляла объект и при недоступном S3 сохраняла строку) и затем были найдены reconciler как 19 stale pending, вместе с одной от теста отказа S3. Лимит S3 поднят до 1G: в повторе 0 рестартов при `memory.peak` ≈ 1 GiB. Этот пик включает page cache записываемых файлов, который ядро может вытеснить.
### Трек 2: внедрение ошибок (grace = 1 мин для эксперимента, 15 мин в стенде)
Методика `scripts/track2/run_track2.py faults`. Периодический reconciler на время теста останавливается. Сбои вносятся напрямую: в S3 через `scripts/track2/s3tool.py` (SigV4 на stdlib, запускается в одноразовом контейнере в сети `data`), в БД через `psql`.
| Сбой (сторона) | Ожидание | dry-run | apply |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| F1 объект без строки (S3) | удалить после grace | orphan_object | удалён |
| F2 потеряна строка неиспользуемого U3 (БД) | объект стал сиротой → удалить | orphan_object | удалён |
| F3 потерян объект используемого U2 (S3) | **метаданные не трогать**, пометить | missing_object | `missing_at` выставлен; GET метаданных 200, content 404, сообщение по-прежнему ссылается на U2 |
| F4 падение между PUT и ready (pending + объект, 1 ч) | удалить строку, затем объект | stale_pending | удалены оба |
| F5 падение до PUT (pending без объекта, 1 ч) | удалить строку | stale_pending | удалена |
| F7 загрузка «в процессе» (свежие pending + объект) | не трогать | не найдена (row известен) | цела |
| F8 свежий объект без строки | не трогать | `skip: younger than grace` | цел |
| U1 используемый, исправный | не трогать | — | скачивается, sha256 верный |
- **Dry-run ничего не меняет:** счётчики S3-объектов, строк, pending и missing до и после совпадают (`dry_run_changed_nothing: true`).
- **Первый apply:** 7 исправлений, 0 ошибок.
- **Повторный apply:** 0 исправлений, только `skip` свежего F8 — проход идемпотентен.
- **Восстановление:** после возврата объекта U2 «из бэкапа» (те же байты) apply выдаёт `recovered=1`, снимает `missing_at`, и скачивание отдаёт исходный sha256.
- **Сторонние находки:** dry-run отдельно учитывает находки вне чата эксперимента (`other_findings_outside_experiment`) — это остатки предыдущих прогонов.
**Почему это безопасно.** Порядок «сначала LIST S3, потом скан БД» плюс «pending-строка пишется до PUT» означают, что у любого объекта из листинга строка уже есть. Перед каждым разрушающим действием есть повторная проверка: сирота — `ObjectKeyKnown` в БД, missing — `StatObject`, pending — `DELETE … WHERE status='pending'`. Удаляются только объекты без строк и pending-строки; ни одна из них не может быть в `message_attachments`, потому что привязать можно только ready. Используемые метаданные никогда не удаляются. Grace 15 мин намного больше бюджета загрузки (60 с). Если `UPDATE → ready` проигрывает гонку с reconciler, загрузка отвечает 503, а не 201.
### Трек 2: стоимость обхода
`scripts/track2/run_track2.py cost`: в БД добавляются N ready-строк (SQL `generate_series`), в S3 — N объектов по 100 B. Каждый размер замеряется трижды в dry-run, после замеров bench-данные удаляются.
| Объектов | list S3, мс | скан БД, мс | всего, мс | мкс/объект | пик памяти reconciler, MiB | находок |
| --- | --- | --- | --- | --- | --- | --- |
| 1 229 | 52 | 6–9 | 59–61 | 48–50 | 10.1–10.9 | 0 |
| 5 229 | 258–345 | 62–70 | 326–408 | 62–78 | 15.8–17.2 | 0 |
| 20 229 | 1321–1537 | 41–95 | 1404–1618 | 69–80 | 30.8–31.2 | 0 |
**Вывод:** стоимость линейна, примерно 70 мкс и 1 КБ RAM на объект. Больше 90% времени занимает постраничный LIST в S3 (по 1000 ключей). На 20k объектов ложных срабатываний 0. При частоте раз в 10 минут нагрузка ничтожна. **Ограничение:** все ключи держатся в памяти, на 10M объектов это ~1 ГБ. Дальше нужен обход по префиксам `attachments/<chat_id>/` или потоковое слияние двух отсортированных списков: ключи S3 и `ORDER BY object_key` в БД.
### Сводка отказов
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
| --- | --- | --- | --- |
| `docker stop` обслужившей API-реплики | LB уводит трафик | первая успешная запись через ~4 с, 0 дублей; файл скачивается через другую | без потерь |
| Restart postgres+valkey+s3+api | данные, сессия и файл живы | PASS, тот же токен работает | без потерь |
| Остановка S3 | 503 только на вложениях | upload/download 503 за 2 с, текст и ready работают | после старта S3 сразу; pending-строк не осталось |
| SeaweedFS убит по памяти (лимит 512M) | 503, без 201 на несохранённое | 18× 503, 2× 201 | 19 pending-строк убраны reconciler; лимит поднят до 1G |
| Внедрённые orphan/missing/pending | найти, исправить безопасно | 5/5 найдены, повторный apply = no-op | missing-байты невосстановимы без бэкапа: метаданные помечены и сохранены |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host, по одному PostgreSQL, S3-узлу и LB: выдерживается отказ одной API-реплики и рестарт стенда, но не потеря хоста или диска S3. В SeaweedFS нет репликации томов.
- Тело загрузки буферизуется в RAM до 10 MiB на слот. Это ограничено и измерено, но не полностью потоковый режим.
- Полное декодирование нужно для отсева повреждённых файлов и стоит CPU: 40 одновременных загрузок 10 МБ занимают ~12 с на 2×1 CPU.
- Объект, у которого обнаружен `missing`, невосстановим без бэкапа. Reconciler только помечает его, и API отдаёт 404 на content.
- Reconciler хранит все ключи в памяти (ограничение указано выше). `size_mismatch` только сообщается, автоматически не исправляется.
- Цель failover-теста выбирается по `X-Instance-Id` обслужившего ответа, а HAProxy раздаёт запросы по кругу, поэтому без вмешательства тест детерминированно останавливал api-2. Для проверки api-1 перед запуском был сделан один лишний запрос через LB (`curl http://localhost:8080/health/live`), сдвигающий очередь; так получен прогон с остановкой api-1.
- CI-job со сборкой стенда и `make test` не добавлен: workflow skeleton проверяет только контракт.
+248
View File
@@ -0,0 +1,248 @@
name: mtusi-lab4
x-api-env: &api-env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_STORE: ${SESSION_STORE:-valkey}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_CACHE_MAX_TTL_SECONDS: ${SESSION_CACHE_MAX_TTL_SECONDS:-60}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
DB_MAX_CONNS: ${DB_MAX_CONNS:-10}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_TIMEOUT: ${S3_TIMEOUT:-15s}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8}
DECODE_BUDGET_BYTES: ${DECODE_BUDGET_BYTES:-134217728}
UPLOAD_TIMEOUT: ${UPLOAD_TIMEOUT:-60s}
GOMEMLIMIT: ${API_GOMEMLIMIT:-400MiB}
services:
postgres:
image: postgres:17.2-alpine3.21
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/pg_password
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
secrets: [pg_password]
volumes:
- pgdata:/var/lib/postgresql/data
networks: [data]
shm_size: 256mb
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1G}"}
stop_grace_period: 30s
restart: unless-stopped
valkey:
image: valkey/valkey:8.0.2-alpine3.21
command: ["valkey-server", "/usr/local/etc/valkey/valkey.conf"]
volumes:
- ./infra/valkey/valkey.conf:/usr/local/etc/valkey/valkey.conf:ro
secrets: [valkey_auth]
networks: [data]
healthcheck:
test: ["CMD-SHELL", "valkey-cli ping | grep -qE 'PONG|NOAUTH'"]
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: unless-stopped
# SeaweedFS (master + volume + filer + S3 gateway in one process). Only the S3
# port is used, and only inside the data network: nothing is published.
s3:
image: chrislusf/seaweedfs:3.80
command:
- server
- -dir=/data
- -s3
- -s3.port=8333
- -s3.config=/run/secrets/s3_identities
- -master.volumeSizeLimitMB=64
- -volume.max=100
secrets: [s3_identities]
volumes:
- s3data:/data
networks: [data]
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
interval: 2s
timeout: 3s
start_period: 60s
retries: 60
deploy:
resources:
# 512M was OOM-killed under 20 parallel 10 MiB PUTs (evidence/memory.json)
limits: {cpus: "1.0", memory: "${S3_MEMORY:-1G}"}
stop_grace_period: 20s
restart: unless-stopped
# Idempotent bucket bootstrap with the admin identity; the API never sees it.
s3-init:
image: mtusi-messenger:lab4
pull_policy: never
command: ["s3-init"]
healthcheck:
disable: true
environment:
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
secrets: [s3_admin_access_key_id, s3_admin_secret_access_key]
networks: [data]
depends_on:
s3:
condition: service_healthy
migrate:
condition: service_completed_successfully
deploy:
resources:
limits: {cpus: "0.5", memory: 64M}
restart: "no"
migrate:
build:
context: .
args:
APP_BUILD: ${APP_BUILD:-dev}
image: mtusi-messenger:lab4
command: ["migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: "no"
api:
image: mtusi-messenger:lab4
pull_policy: never
environment: *api-env
# No writable filesystem: originals cannot end up on the replica's disk.
read_only: true
tmpfs:
- /tmp:size=1m
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [app, data]
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
postgres:
condition: service_healthy
valkey:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 2s
timeout: 2s
start_period: 5s
retries: 3
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"}
stop_grace_period: 15s
restart: unless-stopped
# Periodic DB<->S3 reconciliation (orphan objects, stale pending uploads,
# missing objects). Never touches anything younger than RECONCILE_GRACE.
reconciler:
image: mtusi-messenger:lab4
pull_policy: never
command: ["reconcile", "--apply", "--every", "${RECONCILE_EVERY:-10m}"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
DB_MAX_CONNS: "2"
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
RECONCILE_GRACE: ${RECONCILE_GRACE:-15m}
healthcheck:
disable: true
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
s3-init:
condition: service_completed_successfully
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
restart: unless-stopped
lb:
image: haproxy:3.0.5-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
- "[::1]:${APP_PORT:-8080}:8080"
networks: [app]
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"]
interval: 2s
timeout: 2s
retries: 15
deploy:
resources:
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128M}
restart: unless-stopped
networks:
app: {}
data: {}
volumes:
pgdata: {}
s3data: {}
secrets:
pg_password:
file: ./secrets/pg_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
valkey_auth:
file: ./secrets/valkey_auth.conf
s3_identities:
file: ./secrets/s3_identities.json
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
s3_admin_access_key_id:
file: ./secrets/s3_admin_access_key_id
s3_admin_secret_access_key:
file: ./secrets/s3_admin_secret_access_key
+45
View File
@@ -0,0 +1,45 @@
{
"command": "bootstrap",
"date_utc": "2026-10-29T10:38:50Z",
"commit": "ec540a1",
"result": [
{
"service": "migrate",
"last_log": {
"time": "2026-10-29T10:38:51.949471511Z",
"level": "INFO",
"msg": "migrations up to date",
"total": 3
}
},
{
"service": "s3-init",
"last_log": {
"time": "2026-10-29T10:38:53.253676835Z",
"level": "INFO",
"msg": "bucket ready",
"bucket": "messenger-attachments",
"created": false
}
},
{
"service": "migrate",
"last_log": {
"time": "2026-10-29T10:38:54.658943295Z",
"level": "INFO",
"msg": "migrations up to date",
"total": 3
}
},
{
"service": "s3-init",
"last_log": {
"time": "2026-10-29T10:38:56.030781559Z",
"level": "INFO",
"msg": "bucket ready",
"bucket": "messenger-attachments",
"created": false
}
}
]
}
+4
View File
@@ -0,0 +1,4 @@
## make check
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
## smoke
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+54
View File
@@ -0,0 +1,54 @@
{
"command": "failover-upload",
"date_utc": "2026-10-29T10:38:32Z",
"commit": "ec540a1",
"result": {
"uploaded_via": "0b30fcb99d56",
"stopped_container": "mtusi-lab4-api-2",
"reads_after_stop": [
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
},
{
"instance": "02482efeb4f6",
"sha256_ok": true
}
],
"seconds_observed": 9.6,
"ps_before": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 running\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running",
"ps_during": "mtusi-lab4-api-1 running\nmtusi-lab4-api-2 exited\nmtusi-lab4-lb-1 running\nmtusi-lab4-migrate-1 exited\nmtusi-lab4-postgres-1 running\nmtusi-lab4-reconciler-1 running\nmtusi-lab4-s3-1 running\nmtusi-lab4-s3-init-1 exited\nmtusi-lab4-valkey-1 running"
}
}
+14
View File
@@ -0,0 +1,14 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.095}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
mtusi-lab4-api-1 running
mtusi-lab4-api-2 exited
## other replica (api-1 stopped)
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 4.148}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
mtusi-lab4-api-1 exited
mtusi-lab4-api-2 running
+190
View File
@@ -0,0 +1,190 @@
{
"command": "memory",
"date_utc": "2026-10-29T10:20:43Z",
"commit": "ec540a1",
"result": {
"api_limit": "512M x 2 replicas, 1 CPU each",
"gomemlimit": "400MiB",
"upload_concurrency_per_replica": 8,
"decode_budget_mib": 128,
"s3_limit": "1073741824",
"runs": [
{
"concurrency": 1,
"repeat": 1,
"file_bytes": 10420335,
"ok": 1,
"status_503": 0,
"other": [],
"wall_s": 0.35,
"p50_ms": 337,
"max_ms": 337,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 34.4,
"mtusi-lab4-api-2": 64.4
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 29.5,
"mtusi-lab4-api-2": 54.8
},
"max_peak_mib": 64.4,
"s3_peak_mib_since_start": 267.1,
"s3_restarted_during_run": false
},
{
"concurrency": 1,
"repeat": 2,
"file_bytes": 10420335,
"ok": 1,
"status_503": 0,
"other": [],
"wall_s": 0.36,
"p50_ms": 344,
"max_ms": 344,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 64.5,
"mtusi-lab4-api-2": 29.9
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 54.5,
"mtusi-lab4-api-2": 29.1
},
"max_peak_mib": 64.5,
"s3_peak_mib_since_start": 312.4,
"s3_restarted_during_run": false
},
{
"concurrency": 10,
"repeat": 1,
"file_bytes": 10420335,
"ok": 10,
"status_503": 0,
"other": [],
"wall_s": 4.1,
"p50_ms": 3830,
"max_ms": 3945,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 148.2,
"mtusi-lab4-api-2": 144.4
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 54.8,
"mtusi-lab4-api-2": 30.1
},
"max_peak_mib": 148.2,
"s3_peak_mib_since_start": 960.0,
"s3_restarted_during_run": false
},
{
"concurrency": 10,
"repeat": 2,
"file_bytes": 10420335,
"ok": 10,
"status_503": 0,
"other": [],
"wall_s": 3.05,
"p50_ms": 2844,
"max_ms": 2882,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 143.0,
"mtusi-lab4-api-2": 151.7
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 30.3,
"mtusi-lab4-api-2": 53.4
},
"max_peak_mib": 151.7,
"s3_peak_mib_since_start": 1025.2,
"s3_restarted_during_run": false
},
{
"concurrency": 20,
"repeat": 1,
"file_bytes": 10420335,
"ok": 20,
"status_503": 0,
"other": [],
"wall_s": 6.35,
"p50_ms": 5476,
"max_ms": 6062,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 286.0,
"mtusi-lab4-api-2": 236.4
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 54.8,
"mtusi-lab4-api-2": 29.5
},
"max_peak_mib": 286.0,
"s3_peak_mib_since_start": 1025.2,
"s3_restarted_during_run": false
},
{
"concurrency": 20,
"repeat": 2,
"file_bytes": 10420335,
"ok": 20,
"status_503": 0,
"other": [],
"wall_s": 7.12,
"p50_ms": 6097,
"max_ms": 6830,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 245.4,
"mtusi-lab4-api-2": 263.7
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 30.2,
"mtusi-lab4-api-2": 55.1
},
"max_peak_mib": 263.7,
"s3_peak_mib_since_start": 1025.2,
"s3_restarted_during_run": false
},
{
"concurrency": 40,
"repeat": 1,
"file_bytes": 10420335,
"ok": 40,
"status_503": 0,
"other": [],
"wall_s": 12.64,
"p50_ms": 9590,
"max_ms": 12044,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 264.5,
"mtusi-lab4-api-2": 248.3
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 53.9,
"mtusi-lab4-api-2": 29.9
},
"max_peak_mib": 264.5,
"s3_peak_mib_since_start": 1030.8,
"s3_restarted_during_run": false
},
{
"concurrency": 40,
"repeat": 2,
"file_bytes": 10420335,
"ok": 40,
"status_503": 0,
"other": [],
"wall_s": 11.79,
"p50_ms": 8542,
"max_ms": 11156,
"peak_mib_per_replica": {
"mtusi-lab4-api-1": 260.5,
"mtusi-lab4-api-2": 311.1
},
"baseline_current_mib": {
"mtusi-lab4-api-1": 29.5,
"mtusi-lab4-api-2": 54.5
},
"max_peak_mib": 311.1,
"s3_peak_mib_since_start": 1030.8,
"s3_restarted_during_run": false
}
]
}
}
+29
View File
@@ -0,0 +1,29 @@
{
"command": "no-local-file",
"date_utc": "2026-10-29T10:38:48Z",
"commit": "ec540a1",
"result": {
"mtusi-lab4-api-1": {
"readonly_rootfs": true,
"write_probe": "touch: /app/probe: Read-only file system",
"mounts": [
"/run/secrets/s3_access_key_id",
"/run/secrets/s3_secret_access_key",
"/run/secrets/database_url",
"/run/secrets/cache_url"
],
"files_written_since_start": []
},
"mtusi-lab4-api-2": {
"readonly_rootfs": true,
"write_probe": "touch: /app/probe: Read-only file system",
"mounts": [
"/run/secrets/s3_access_key_id",
"/run/secrets/s3_secret_access_key",
"/run/secrets/database_url",
"/run/secrets/cache_url"
],
"files_written_since_start": []
}
}
}
+21
View File
@@ -0,0 +1,21 @@
api running Up About a minute (healthy)
api running Up About a minute (healthy)
lb running Up 11 minutes (healthy)
migrate exited Exited (0) About a minute ago
postgres running Up 11 minutes (healthy)
reconciler running Up About a minute
s3 running Up 38 seconds (healthy)
s3-init exited Exited (0) About a minute ago
valkey running Up 11 minutes (healthy)
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
api running Up 7 seconds (healthy)
api running Up 7 seconds (healthy)
lb running Up 1 second (health: starting)
migrate exited Exited (0) 8 seconds ago
postgres running Up 46 seconds (healthy)
reconciler running Up 7 seconds
s3 running Up 36 seconds (healthy)
s3-init exited Exited (0) 8 seconds ago
valkey running Up 46 seconds (healthy)
+19
View File
@@ -0,0 +1,19 @@
{
"command": "private",
"date_utc": "2026-10-29T10:38:17Z",
"commit": "ec540a1",
"result": {
"object_key_shape": "attachments/<chat_id>/<attachment_id> (server generated)",
"key_contains_filename": false,
"anonymous_get": 403,
"anonymous_list": "denied (HTTP 403)",
"api_identity_get": 200,
"api_identity_get_sha256_matches": true,
"api_identity_create_bucket": 403,
"anonymous_create_bucket": 403,
"published_s3_ports": "invalid IP:0",
"outsider_via_api_metadata": 404,
"outsider_via_api_content": 404,
"api_identity_put_other_bucket": "{\"cmd\": \"put\", \"as\": \"api\", \"status\": 403}"
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"command": "s3-outage",
"date_utc": "2026-10-29T11:45:59Z",
"commit": "ec540a1",
"result": {
"upload_status": 503,
"upload_ms": 2048,
"download_status": 503,
"download_ms": 2018,
"metadata_status": 200,
"text_message_status": 201,
"ready_status": 200,
"pending_rows_left": 0,
"upload_recovered_after_s3_healthy_s": 0.1,
"old_file_intact": true
}
}
+92
View File
@@ -0,0 +1,92 @@
[
{
"test": "one session served by >=2 replicas without re-login (no stickiness)",
"result": "PASS",
"detail": "distribution [15, 15]"
},
{
"test": "X-Instance-Id is present on errors and health too",
"result": "PASS",
"detail": null
},
{
"test": "Idempotency-Key missing or malformed -> 400, nothing stored",
"result": "PASS",
"detail": null
},
{
"test": "concurrent replay of one key through different replicas -> exactly one message",
"result": "PASS",
"detail": "24 parallel requests via 2 replicas -> 1 message"
},
{
"test": "replay is semantic: JSON key order/whitespace does not matter; other body -> 409",
"result": "PASS",
"detail": null
},
{
"test": "key scope = user + chat: same key is independent for other user / other chat",
"result": "PASS",
"detail": null
},
{
"test": "logout is visible on every replica immediately (shared sessions)",
"result": "PASS",
"detail": "revoked token rejected by 2 replicas"
},
{
"test": "replay re-checks the session and chat access",
"result": "PASS",
"detail": null
},
{
"test": "PNG and JPEG upload/download byte-exact with true Content-Type",
"result": "PASS",
"detail": "png 111817 B, jpeg 2565 B"
},
{
"test": "size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413",
"result": "PASS",
"detail": "accepted 10420335 B, rejected 10485761 B"
},
{
"test": "decompression bomb (400M px header, tiny body) -> 413 before decoding",
"result": "PASS",
"detail": "83 B rejected in 25 ms"
},
{
"test": "corrupted images and MIME spoofing -> 415, never 500",
"result": "PASS",
"detail": "9 spoof/corrupt variants -> 415"
},
{
"test": "dangerous filenames -> 400; Unicode display name kept as-is",
"result": "PASS",
"detail": "12 bad names rejected"
},
{
"test": "attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400",
"result": "PASS",
"detail": null
},
{
"test": "idempotency: missing attachment_ids == []; other attachment list -> 409",
"result": "PASS",
"detail": null
},
{
"test": "10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back",
"result": "PASS",
"detail": "10 uploads via 2 replicas"
},
{
"test": "ACL: outsider gets 404 on upload, metadata, content; no token -> 401",
"result": "PASS",
"detail": null
},
{
"test": "message attachment_ids: other chat / foreign / unknown -> 404, nothing stored",
"result": "PASS",
"detail": null
}
]
+19
View File
@@ -0,0 +1,19 @@
PASS one session served by >=2 replicas without re-login (no stickiness) - distribution [15, 15]
PASS X-Instance-Id is present on errors and health too -
PASS Idempotency-Key missing or malformed -> 400, nothing stored -
PASS concurrent replay of one key through different replicas -> exactly one message - 24 parallel requests via 2 replicas -> 1 message
PASS replay is semantic: JSON key order/whitespace does not matter; other body -> 409 -
PASS key scope = user + chat: same key is independent for other user / other chat -
PASS logout is visible on every replica immediately (shared sessions) - revoked token rejected by 2 replicas
PASS replay re-checks the session and chat access -
PASS PNG and JPEG upload/download byte-exact with true Content-Type - png 111817 B, jpeg 2565 B
PASS size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413 - accepted 10420335 B, rejected 10485761 B
PASS decompression bomb (400M px header, tiny body) -> 413 before decoding - 83 B rejected in 25 ms
PASS corrupted images and MIME spoofing -> 415, never 500 - 9 spoof/corrupt variants -> 415
PASS dangerous filenames -> 400; Unicode display name kept as-is - 12 bad names rejected
PASS attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400 -
PASS idempotency: missing attachment_ids == []; other attachment list -> 409 -
PASS 10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back - 10 uploads via 2 replicas
PASS ACL: outsider gets 404 on upload, metadata, content; no token -> 401 -
PASS message attachment_ids: other chat / foreign / unknown -> 404, nothing stored -
18/18 passed
+120
View File
@@ -0,0 +1,120 @@
{
"command": "cost",
"date_utc": "2026-10-29T10:33:43Z",
"commit": "ec540a1",
"result": {
"object_size_bytes": 100,
"reconciler_limits": "0.5 CPU, 128M",
"s3": "SeaweedFS 3.80, 1 CPU, 1G",
"runs": [
{
"bench_objects": 1000,
"repeat": 1,
"objects_scanned": 1229,
"rows_scanned": 1232,
"list_ms": 52,
"db_ms": 9,
"total_ms": 61,
"per_object_us": 50.0,
"container_memory_peak_mib": 10.7,
"findings": 0
},
{
"bench_objects": 1000,
"repeat": 2,
"objects_scanned": 1229,
"rows_scanned": 1232,
"list_ms": 52,
"db_ms": 6,
"total_ms": 59,
"per_object_us": 48.1,
"container_memory_peak_mib": 10.1,
"findings": 0
},
{
"bench_objects": 1000,
"repeat": 3,
"objects_scanned": 1229,
"rows_scanned": 1232,
"list_ms": 52,
"db_ms": 7,
"total_ms": 60,
"per_object_us": 48.8,
"container_memory_peak_mib": 10.9,
"findings": 0
},
{
"bench_objects": 5000,
"repeat": 1,
"objects_scanned": 5229,
"rows_scanned": 5232,
"list_ms": 258,
"db_ms": 67,
"total_ms": 326,
"per_object_us": 62.5,
"container_memory_peak_mib": 17.2,
"findings": 0
},
{
"bench_objects": 5000,
"repeat": 2,
"objects_scanned": 5229,
"rows_scanned": 5232,
"list_ms": 345,
"db_ms": 62,
"total_ms": 408,
"per_object_us": 78.2,
"container_memory_peak_mib": 16.9,
"findings": 0
},
{
"bench_objects": 5000,
"repeat": 3,
"objects_scanned": 5229,
"rows_scanned": 5232,
"list_ms": 263,
"db_ms": 70,
"total_ms": 335,
"per_object_us": 64.1,
"container_memory_peak_mib": 15.8,
"findings": 0
},
{
"bench_objects": 20000,
"repeat": 1,
"objects_scanned": 20229,
"rows_scanned": 20232,
"list_ms": 1425,
"db_ms": 95,
"total_ms": 1523,
"per_object_us": 75.3,
"container_memory_peak_mib": 30.8,
"findings": 0
},
{
"bench_objects": 20000,
"repeat": 2,
"objects_scanned": 20229,
"rows_scanned": 20232,
"list_ms": 1537,
"db_ms": 41,
"total_ms": 1618,
"per_object_us": 80.0,
"container_memory_peak_mib": 31.1,
"findings": 0
},
{
"bench_objects": 20000,
"repeat": 3,
"objects_scanned": 20229,
"rows_scanned": 20232,
"list_ms": 1321,
"db_ms": 76,
"total_ms": 1404,
"per_object_us": 69.4,
"container_memory_peak_mib": 31.2,
"findings": 0
}
]
}
}
+244
View File
@@ -0,0 +1,244 @@
{
"command": "faults",
"date_utc": "2026-10-29T10:31:36Z",
"commit": "ec540a1",
"result": {
"grace_used": "1m",
"faults": {
"F1_orphan_object_no_row": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
"F2_metadata_row_lost(U3)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
"F3_object_lost_of_used(U2)": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
"F4_crash_after_put_before_ready": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
"F5_crash_before_put": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
"F6_F7_fresh_upload_in_progress": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/693c91e3-43f6-46f6-9bef-2b0de987546a",
"F8_fresh_orphan_object": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc"
},
"steps": [
{
"step": "dry-run",
"report": {
"mode": "dry-run",
"objects_scanned": 233,
"rows_scanned": 235,
"orphan_objects": 3,
"fresh_skipped": 1,
"stale_pending": 3,
"missing_objects": 1,
"size_mismatch": 0,
"recovered": 0,
"fixed": 0,
"errors": 0,
"total_ms": 34
},
"items": [
{
"kind": "stale_pending",
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
"attachment_id": "34bf2792-f4dd-4fc8-b242-2f2ed7b49209",
"action": "would delete row and object"
},
{
"kind": "stale_pending",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
"action": "would delete row and object"
},
{
"kind": "stale_pending",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
"action": "would delete row and object"
},
{
"kind": "missing_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
"action": "would mark missing (metadata kept)"
},
{
"kind": "orphan_object",
"key": "attachments/22b62cf4-b6c8-4cfb-bd41-0a248cef291e/5585fa4c-c149-49f6-b5d1-1196038eba43",
"action": "would delete object"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
"action": "skip: younger than grace"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
"action": "would delete object"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
"action": "would delete object"
}
],
"state_before": {
"s3_objects": 233,
"db_rows": 235,
"db_pending": 4,
"db_missing_marked": 4
},
"state_after": {
"s3_objects": 233,
"db_rows": 235,
"db_pending": 4,
"db_missing_marked": 4
},
"dry_run_changed_nothing": true,
"this_experiment": {
"stale_pending": 2,
"missing_object": 1,
"orphan_object/skip": 1,
"orphan_object": 2
},
"other_findings_outside_experiment": {
"stale_pending": 1,
"orphan_object": 1
}
},
{
"step": "apply #1",
"report": {
"mode": "apply",
"objects_scanned": 233,
"rows_scanned": 235,
"orphan_objects": 3,
"fresh_skipped": 1,
"stale_pending": 3,
"missing_objects": 1,
"size_mismatch": 0,
"recovered": 0,
"fixed": 7,
"errors": 0,
"total_ms": 46
},
"this_experiment": {
"stale_pending": 2,
"missing_object": 1,
"orphan_object/skip": 1,
"orphan_object": 2
},
"items": [
{
"kind": "stale_pending",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/761f488e-6a97-417b-a96b-140f3f99660a",
"attachment_id": "761f488e-6a97-417b-a96b-140f3f99660a",
"action": "deleted row and object"
},
{
"kind": "stale_pending",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
"attachment_id": "63fe6ccb-a129-4f4d-8ef5-f56a57ae21b5",
"action": "deleted row and object"
},
{
"kind": "missing_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/421fa744-895f-4f52-96cf-ea62ffec57ad",
"attachment_id": "421fa744-895f-4f52-96cf-ea62ffec57ad",
"action": "marked missing (metadata kept)"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/3b644cf0-c1cd-42e1-82e4-870552ba02bc",
"action": "skip: younger than grace"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/9bd74768-2cca-40cb-9f66-f9779c7e1bed",
"action": "deleted object"
},
{
"kind": "orphan_object",
"key": "attachments/4a583714-f34c-418f-816d-a37c4667a15e/90004716-e003-45b9-aca5-1043ee5f2a69",
"action": "deleted object"
}
],
"state_after": {
"s3_objects": 228,
"db_rows": 232,
"db_pending": 1,
"db_missing_marked": 5
}
},
{
"step": "apply #2 (repeat)",
"report": {
"mode": "apply",
"objects_scanned": 228,
"rows_scanned": 232,
"orphan_objects": 0,
"fresh_skipped": 1,
"stale_pending": 0,
"missing_objects": 0,
"size_mismatch": 0,
"recovered": 0,
"fixed": 0,
"errors": 0,
"total_ms": 25
},
"this_experiment": {
"orphan_object/skip": 1
},
"state_after": {
"s3_objects": 228,
"db_rows": 232,
"db_pending": 1,
"db_missing_marked": 5
}
},
{
"step": "apply after restoring U2 object",
"report": {
"mode": "apply",
"objects_scanned": 229,
"rows_scanned": 232,
"orphan_objects": 0,
"fresh_skipped": 1,
"stale_pending": 0,
"missing_objects": 0,
"size_mismatch": 0,
"recovered": 1,
"fixed": 1,
"errors": 0,
"total_ms": 27
}
},
{
"step": "dry-run with production grace 15m",
"report": {
"mode": "dry-run",
"objects_scanned": 229,
"rows_scanned": 232,
"orphan_objects": 0,
"fresh_skipped": 1,
"stale_pending": 0,
"missing_objects": 0,
"size_mismatch": 0,
"recovered": 0,
"fixed": 0,
"errors": 0,
"total_ms": 31
}
}
],
"verify": {
"U1_used_healthy_download_sha_ok": true,
"U2_metadata_kept": 200,
"U2_content_status": 404,
"U2_marked_missing": true,
"message_still_references_U1_U2": true,
"F1_orphan_deleted": true,
"F2_orphan_of_lost_row_deleted": true,
"F4_row_and_object_deleted": true,
"F5_row_deleted": true,
"F7_fresh_pending_row_kept": true,
"F7_fresh_object_kept": true,
"F8_fresh_orphan_kept": true,
"U2_restored_and_unmarked": true
}
}
}
+45
View File
@@ -0,0 +1,45 @@
global
log stdout format raw local0 info
maxconn 4096
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold obsolete 5s
accepted_payload_size 8192
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 1s
timeout client 75s
timeout server 75s
timeout http-request 5s
timeout http-keep-alive 10s
timeout check 500ms
retries 2
retry-on conn-failure
option redispatch 1
frontend public
bind :8080
default_backend api
backend api
balance roundrobin
option httpchk GET /health/ready
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 init-addr none resolvers docker resolve-prefer ipv4
server-template api 8 api:8080
frontend internal_stats
bind :8404
monitor-uri /healthz
stats enable
stats uri /stats
stats refresh 5s
+8
View File
@@ -0,0 +1,8 @@
bind 0.0.0.0
port 6379
protected-mode yes
save ""
appendonly no
maxmemory 64mb
maxmemory-policy volatile-ttl
include /run/secrets/valkey_auth
+56
View File
@@ -0,0 +1,56 @@
#!/usr/bin/env python3
import json
import secrets
import shutil
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / 'secrets'
def write(name, value):
path = SECRETS / name
if path.exists():
print('keep secrets/' + name)
return path.read_text().strip()
path.write_bytes((value + '\n').encode('utf-8'))
print('created secrets/' + name)
return value
def main():
SECRETS.mkdir(exist_ok=True)
pg = write('pg_password', secrets.token_urlsafe(24))
write('database_url', 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable' % pg)
vk_conf = SECRETS / 'valkey_auth.conf'
if vk_conf.exists():
vk = vk_conf.read_text().split()[-1]
print('keep secrets/valkey_auth.conf')
else:
vk = secrets.token_urlsafe(24)
write('valkey_auth.conf', 'requirepass ' + vk)
write('cache_url', 'redis://:%s@valkey:6379/0' % vk)
# Two S3 identities: admin (bucket bootstrap only) and api (read/write/list
# of the one bucket). SeaweedFS has no anonymous identity -> bucket is private.
admin_id = write('s3_admin_access_key_id', 'admin' + secrets.token_hex(8))
admin_secret = write('s3_admin_secret_access_key', secrets.token_urlsafe(30))
api_id = write('s3_access_key_id', 'api' + secrets.token_hex(8))
api_secret = write('s3_secret_access_key', secrets.token_urlsafe(30))
bucket = 'messenger-attachments'
identities = {'identities': [
{'name': 'bootstrap-admin',
'credentials': [{'accessKey': admin_id, 'secretKey': admin_secret}],
'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']},
{'name': 'messenger-api',
'credentials': [{'accessKey': api_id, 'secretKey': api_secret}],
'actions': ['Read:' + bucket, 'Write:' + bucket, 'List:' + bucket]},
]}
write('s3_identities.json', json.dumps(identities, indent=2))
env = ROOT / '.env'
if not env.exists():
shutil.copy(ROOT / '.env.example', env)
print('created .env from .env.example')
if __name__ == '__main__':
main()
+269
View File
@@ -0,0 +1,269 @@
#!/usr/bin/env python3
"""Operator checks for lab4 that need Docker access (run on the host, explicitly).
python scripts/lab4ops.py private # bucket privacy outside the API
python scripts/lab4ops.py failover-upload # upload -> stop that replica -> download via the other
python scripts/lab4ops.py s3-outage # stop S3: uploads/downloads 503, text path keeps working
python scripts/lab4ops.py no-local-file # API containers have no volume and no stored original
python scripts/lab4ops.py bootstrap # migrate + bucket bootstrap are idempotent
python scripts/lab4ops.py memory # peak RSS under concurrent uploads
Each command prints JSON and, with --output, stores it in evidence/.
Synthetic data only; tokens/credentials are never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check # noqa: E402
import imagegen # noqa: E402
from lablib import api_containers, sh, wait_healthy # noqa: E402
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
NETWORK = 'mtusi-lab4_data'
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
def winpath(p):
s = str(p)
return s.replace('\\', '/')
def s3tool(*args, stdin=None):
"""Run s3tool.py in a throwaway container on the data network (S3 is not published)."""
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
r = subprocess.run(cmd, input=stdin, capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('s3tool %s failed: %s' % (args[0], (r.stdout + r.stderr).strip()[-300:]))
return json.loads(r.stdout.strip().splitlines()[-1])
def psql(sql):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-v', 'ON_ERROR_STOP=1', '-c', sql)
def upload(c, actor, chat, data=PNG, name='sample.png', expected=201):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
actor=actor, raw=data, headers={'Content-Type': 'image/png'},
schema='Attachment' if expected == 201 else None)
def cmd_private(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % att['id'])
res = {
'object_key_shape': 'attachments/<chat_id>/<attachment_id> (server generated)',
'key_contains_filename': 'sample' in key,
'anonymous_get': s3tool('get', key, '--as', 'anonymous')['status'],
'anonymous_list': None,
'api_identity_get': s3tool('get', key)['status'],
'api_identity_get_sha256_matches': s3tool('get', key)['sha256'] == att['sha256'],
'api_identity_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket')['status'],
'anonymous_create_bucket': s3tool('create-bucket', 'lab4-forbidden-bucket', '--as', 'anonymous')['status'],
'published_s3_ports': sh('docker', 'compose', 'port', 's3', '8333', check=False) or None,
'outsider_via_api_metadata': c.call('GET', '/api/v1/attachments/' + att['id'], 404, actor=eve)['status'],
'outsider_via_api_content': c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 404, actor=eve)['status'],
}
try:
s3tool('list', 'attachments/', '--as', 'anonymous')
res['anonymous_list'] = 200
except RuntimeError as exc:
res['anonymous_list'] = 'denied (%s)' % str(exc).split(':')[-1].strip()
# the API identity is bound to ONE bucket: the admin creates another one, the API cannot write there
s3tool('create-bucket', 'lab4-other-bucket', '--as', 'admin')
res['api_identity_put_other_bucket'] = subprocess.run(
['docker', 'run', '--rm', '-e', 'S3_BUCKET=lab4-other-bucket', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', 'probe'],
capture_output=True, text=True).stdout.strip()
check(res['anonymous_get'] == 403 and res['api_identity_get'] == 200, 'unexpected bucket privacy')
check(res['api_identity_create_bucket'] == 403 and '"status": 403' in res['api_identity_put_other_bucket'],
'api identity has more rights than expected')
return res
def cmd_failover_upload(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
r = upload(c, alice, chat)
att, served_by = r['json'], r['headers']['X-Instance-Id']
target = [x for x in api_containers() if x['id'].startswith(served_by[:12])]
check(len(target) == 1, 'cannot map X-Instance-Id to one api container')
before = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
sh('docker', 'stop', '-t', '15', target[0]['id'])
t0 = time.monotonic()
reads = []
try:
for _ in range(10):
try:
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
reads.append({'instance': d['headers']['X-Instance-Id'],
'sha256_ok': hashlib.sha256(d['bytes']).hexdigest() == att['sha256']})
except Failure as exc:
reads.append({'error': str(exc)[-60:]})
time.sleep(0.3)
during = sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}} {{.State}}')
finally:
sh('docker', 'start', target[0]['id'])
wait_healthy(target[0]['id'])
ok = [x for x in reads if x.get('sha256_ok')]
check(len(ok) >= 8 and all(x['instance'] != served_by for x in ok), 'download did not move to the other replica')
return {'uploaded_via': served_by, 'stopped_container': target[0]['name'], 'reads_after_stop': reads,
'seconds_observed': round(time.monotonic() - t0, 1), 'ps_before': before, 'ps_during': during}
def cmd_s3_outage(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att = upload(c, alice, chat)['json']
pending_before = psql("SELECT count(*) FROM attachments WHERE status = 'pending'")
sh('docker', 'compose', 'stop', 's3')
res = {}
try:
u = upload(c, alice, chat, expected=503)
res['upload_status'], res['upload_ms'] = u['status'], round(u['elapsed'] * 1000)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob)
res['download_status'], res['download_ms'] = d['status'], round(d['elapsed'] * 1000)
res['metadata_status'] = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob)['status']
res['text_message_status'] = c.message(alice, chat, 'S3 лежит, текст работает')['status']
res['ready_status'] = c.call('GET', '/health/ready')['status']
# expected 0: compensation deletes the pending row; a possible object is an orphan for the reconciler
res['pending_rows_left'] = int(psql(
"SELECT count(*) FROM attachments WHERE status = 'pending'")) - int(pending_before)
finally:
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 's3')
t0 = time.monotonic()
while True:
try:
upload(c, alice, chat)
break
except Failure:
check(time.monotonic() - t0 < 60, 'uploads did not recover after S3 start')
time.sleep(0.5)
res['upload_recovered_after_s3_healthy_s'] = round(time.monotonic() - t0, 1)
d = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
res['old_file_intact'] = hashlib.sha256(d['bytes']).hexdigest() == att['sha256']
check(res['upload_status'] == 503 and res['text_message_status'] == 201 and res['ready_status'] == 200,
'S3 outage is not isolated to attachment endpoints')
return res
def cmd_no_local_file(a):
out = {}
for r in api_containers():
mounts = json.loads(sh('docker', 'inspect', '-f', '{{json .Mounts}}', r['id']))
ro = sh('docker', 'inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', r['id'])
probe = sh('docker', 'exec', r['id'], 'sh', '-c', 'touch /app/probe 2>&1 || true')
# Docker itself bind-mounts /etc/hosts, /etc/hostname, /etc/resolv.conf
files = sh('docker', 'exec', r['id'], 'sh', '-c',
'find / -xdev -type f -newer /app/server ! -path "/proc/*" ! -path "/sys/*" '
'! -path "/run/secrets/*" ! -name hosts ! -name hostname ! -name resolv.conf '
'! -name .dockerenv 2>/dev/null | head -20')
out[r['name']] = {'readonly_rootfs': ro == 'true', 'write_probe': probe,
'mounts': [m.get('Destination') for m in mounts],
'files_written_since_start': files.splitlines()}
check(ro == 'true' and not files.strip(), 'API container can store files locally')
return out
def cmd_bootstrap(a):
runs = []
for _ in range(2):
for svc in ('migrate', 's3-init'):
log = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', svc)
runs.append({'service': svc, 'last_log': json.loads(log.strip().splitlines()[-1])})
return runs
def peak(cid):
v = sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory.current').split()
return int(v[0]) / 2**20, int(v[1]) / 2**20
def cmd_memory(a):
big = imagegen.near_limit_png()
c = Client(a.base_url, timeout=120)
alice, bob, eve, chat = c.bootstrap()
results = []
for conc in a.levels:
for rep in range(a.repeat):
sh('docker', 'compose', 'restart', 'api')
for r in api_containers():
check(wait_healthy(r['id'], 60), 'api not healthy')
time.sleep(1)
s3_id = sh('docker', 'compose', 'ps', '-q', 's3')
s3_restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id)
upload(c, alice, chat, big) # warm-up allocations of one replica
base = {r['name']: peak(r['id']) for r in api_containers()}
barrier = threading.Barrier(conc)
def one(i):
local = Client(a.base_url, timeout=120)
barrier.wait()
try:
x = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=big.png',
[201, 503], actor=alice, raw=big, headers={'Content-Type': 'image/png'})
return x['status'], x['elapsed']
except Failure as exc:
return str(exc)[-40:], 0
t0 = time.monotonic()
with ThreadPoolExecutor(conc) as pool:
res = list(pool.map(one, range(conc)))
wall = time.monotonic() - t0
after = {r['name']: peak(r['id']) for r in api_containers()}
lat = sorted(e for s, e in res if s == 201)
row = {'concurrency': conc, 'repeat': rep + 1, 'file_bytes': len(big),
'ok': sum(1 for s, _ in res if s == 201), 'status_503': sum(1 for s, _ in res if s == 503),
'other': [s for s, _ in res if s not in (201, 503)], 'wall_s': round(wall, 2),
'p50_ms': round(1000 * lat[len(lat) // 2]) if lat else None,
'max_ms': round(1000 * lat[-1]) if lat else None,
'peak_mib_per_replica': {k: round(v[0], 1) for k, v in after.items()},
'baseline_current_mib': {k: round(v[1], 1) for k, v in base.items()}}
row['max_peak_mib'] = max(row['peak_mib_per_replica'].values())
row['s3_peak_mib_since_start'] = round(peak(s3_id)[0], 1)
row['s3_restarted_during_run'] = sh('docker', 'inspect', '-f', '{{.RestartCount}}', s3_id) != s3_restarts
print(json.dumps(row, ensure_ascii=False), flush=True)
results.append(row)
return {'api_limit': '512M x 2 replicas, 1 CPU each', 'gomemlimit': '400MiB',
'upload_concurrency_per_replica': 8, 'decode_budget_mib': 128,
's3_limit': sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 's3')),
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['private', 'failover-upload', 's3-outage', 'no-local-file', 'bootstrap', 'memory'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
p.add_argument('--levels', type=int, nargs='+', default=[1, 10, 20, 40])
p.add_argument('--repeat', type=int, default=2)
a = p.parse_args()
fn = globals()['cmd_' + a.cmd.replace('-', '_')]
result = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False), 'result': fn(a)}
text = json.dumps(result, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(a.output).write_text(text + '\n', encoding='utf-8')
print('PASS: ' + a.cmd)
if __name__ == '__main__':
main()
+130
View File
@@ -0,0 +1,130 @@
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
PROJECT = 'mtusi-lab4'
def sh(*args, check=True):
r = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8')
if check and r.returncode != 0:
raise RuntimeError('%s failed: %s' % (' '.join(args[:3]), r.stderr.strip()[:300]))
return r.stdout.strip()
def api_containers(all_states=False):
out = sh('docker', 'ps', *(['-a'] if all_states else []), '--no-trunc',
'--filter', 'label=com.docker.compose.project=' + PROJECT,
'--filter', 'label=com.docker.compose.service=api',
'--format', '{{json .}}')
rows = []
for line in out.splitlines():
d = json.loads(line)
rows.append({'id': d['ID'][:12], 'name': d['Names'], 'state': d['State'], 'status': d['Status']})
return sorted(rows, key=lambda r: r['name'])
def health(cid):
return sh('docker', 'inspect', '-f', '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}', cid)
def wait_healthy(cid, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if health(cid) == 'healthy':
return True
time.sleep(0.25)
return False
def ps_snapshot():
return sh('docker', 'compose', 'ps', '-a', '--format', '{{.Name}}\t{{.State}}\t{{.Status}}')
class WriteLoad:
def __init__(self, base_url, workers=8, timeout=2.0, pause=0.0):
self.base = base_url
self.workers = workers
self.timeout = timeout
self.pause = pause
self.events = []
self.confirmed = []
self.retries = 0
self._lock = threading.Lock()
self._stop = threading.Event()
self._threads = []
c = Client(base_url, timeout=10)
self.alice, self.bob, _eve, self.chat = c.bootstrap()
self.reader = c
def _worker(self, wid):
c = Client(self.base, timeout=self.timeout)
key = text = None
n = 0
while not self._stop.is_set():
if key is None:
key = uuid.uuid4().hex
text = 'load-%d-%d-%s' % (wid, n, key[:8])
start = time.monotonic()
try:
r = c.message(self.alice, self.chat, text, key=key)
with self._lock:
self.events.append((start, True, r['headers']['X-Instance-Id'], time.monotonic() - start))
self.confirmed.append(r['json'])
key = None
n += 1
except Failure as exc:
with self._lock:
self.events.append((start, False, str(exc)[-60:], time.monotonic() - start))
self.retries += 1
time.sleep(0.05)
if self.pause:
time.sleep(self.pause)
def start(self):
for i in range(self.workers):
t = threading.Thread(target=self._worker, args=(i,), daemon=True)
t.start()
self._threads.append(t)
self.t0 = time.monotonic()
def stop(self):
self._stop.set()
for t in self._threads:
t.join(timeout=15)
def window(self, a, b):
ev = [e for e in self.events if a <= e[0] < b]
ok = [e for e in ev if e[1]]
lat = sorted(e[3] for e in ok)
pct = lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1) if lat else None
return {'seconds': round(b - a, 2), 'ok': len(ok), 'errors': len(ev) - len(ok),
'ok_rps': round(len(ok) / (b - a), 1) if b > a else None,
'p50_ms': pct(.5), 'p95_ms': pct(.95), 'p99_ms': pct(.99)}
def verify_history(self):
items, cursor = [], None
path = '/api/v1/chats/' + self.chat['id'] + '/messages?limit=100'
while True:
page = self.reader.call('GET', path + ('&cursor=' + cursor if cursor else ''),
actor=self.bob, schema='MessagePage')['json']
items += page['items']
cursor = page['next_cursor']
if not cursor:
break
by_id = {m['id']: m for m in items}
texts = [m['text'] for m in items]
lost = [m for m in self.confirmed if by_id.get(m['id']) != m]
return {'stored_messages': len(items), 'confirmed_201': len(self.confirmed),
'lost_confirmed': len(lost), 'duplicate_ids': len(items) - len(by_id),
'duplicate_texts': len(texts) - len(set(texts)),
'unique_confirmed': len({m['id'] for m in self.confirmed})}
+6
View File
@@ -0,0 +1,6 @@
@echo off
cd /d "%~dp0.."
docker compose stop lb api reconciler || exit /b 1
docker compose restart postgres valkey s3 || exit /b 1
docker compose up -d --no-recreate --wait postgres valkey s3 || exit /b 1
docker compose start api reconciler lb || exit /b 1
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
# Restart API and ALL state stores (PostgreSQL, Valkey, S3) keeping volumes.
set -eu
cd "$(dirname "$0")/.."
docker compose stop lb api reconciler
docker compose restart postgres valkey s3
# no recreation, only waits until the stores report healthy again
docker compose up -d --no-recreate --wait postgres valkey s3
docker compose start api reconciler lb
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
echo %TARGET_INSTANCE_ID%| findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
set CID=
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=mtusi-lab4" --filter "label=com.docker.compose.service=api"') do set CID=%%i
if "%CID%"=="" exit /b 3
docker stop -t 15 %CID% >nul || exit /b 1
echo stopped api container %CID%
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]*) echo "unexpected instance id format" >&2; exit 2 ;;
esac
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=mtusi-lab4" \
--filter "label=com.docker.compose.service=api")
[ -n "$cid" ] || { echo "no running api replica matches" >&2; exit 3; }
[ "$(printf '%s\n' "$cid" | wc -l)" -eq 1 ] || { echo "ambiguous id" >&2; exit 3; }
docker stop -t 15 "$cid" >/dev/null
echo "stopped api container $cid"
+254
View File
@@ -0,0 +1,254 @@
#!/usr/bin/env python3
"""Track 2: DB<->S3 consistency. Fault injection on both sides + reconciler cost.
python scripts/track2/run_track2.py faults --output evidence/track2/faults.json
python scripts/track2/run_track2.py cost --sizes 1000 5000 20000 --repeat 3 --output evidence/track2/cost.json
The periodic `reconciler` service is stopped for the duration (it would race the
experiment) and started again at the end. Synthetic data only.
"""
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'scripts'))
from lab4ops import NETWORK, PNG, TOOL_IMAGE, psql, s3tool, upload, winpath # noqa: E402
from lablib import sh # noqa: E402
from client import Client, check # noqa: E402
def s3put_bytes(key, data):
cmd = ['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro', '-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', 'put', key, '--stdin']
r = subprocess.run(cmd, input=data, capture_output=True)
check(r.returncode == 0, 's3 put failed')
return json.loads(r.stdout.decode().strip().splitlines()[-1])
def reconcile(*args):
"""One reconciler pass in a throwaway container; also returns its cgroup memory peak."""
out = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 'reconciler', '-c',
'/app/server reconcile ' + ' '.join(args) + ' > /tmp/r.json && cat /tmp/r.json '
'&& echo && cat /sys/fs/cgroup/memory.peak')
lines = [x for x in out.splitlines() if x.strip()]
rep = json.loads(lines[-2])
rep['container_memory_peak_mib'] = round(int(lines[-1]) / 2**20, 1)
return rep
def counts(prefix='attachments/'):
return {'s3_objects': s3tool('count', prefix)['count'],
'db_rows': int(psql('SELECT count(*) FROM attachments')),
'db_pending': int(psql("SELECT count(*) FROM attachments WHERE status = 'pending'")),
'db_missing_marked': int(psql('SELECT count(*) FROM attachments WHERE missing_at IS NOT NULL'))}
def summary(rep):
keys = ('mode', 'objects_scanned', 'rows_scanned', 'orphan_objects', 'fresh_skipped', 'stale_pending',
'missing_objects', 'size_mismatch', 'recovered', 'fixed', 'errors', 'total_ms')
return {k: rep[k] for k in keys}
def scoped(rep, prefix, invert=False):
out = {}
for i in rep['items']:
if i.get('key', '').startswith(prefix) == invert:
continue
k = i['kind'] + ('/skip' if i['action'].startswith('skip') else '')
out[k] = out.get(k, 0) + 1
return out
def insert_pending(att_id, chat, owner, key, age):
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height,"
" object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'crash.png', 'image/png', 100,"
" repeat('0', 64), 1, 1, '%s', 'pending', now() - interval '%s', now() - interval '%s')"
% (att_id, chat, owner, key, age, age))
def exists(key):
return s3tool('get', key)['status'] == 200
def cmd_faults(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
cid, prefix = chat['id'], 'attachments/' + chat['id'] + '/'
key_of = lambda att_id: psql("SELECT object_key FROM attachments WHERE id = '%s'" % att_id)
# healthy, referenced attachment (must survive everything)
u1 = upload(c, alice, chat)['json']
# referenced attachment whose object will be lost (S3-side fault)
u2 = upload(c, alice, chat)['json']
msg = c.message(alice, chat, '', attachments=[u1['id'], u2['id']])['json']
# unreferenced ready attachment whose metadata row will be lost (DB-side fault)
u3 = upload(c, alice, chat)['json']
u3_key = key_of(u3['id'])
f = {}
f['F1_orphan_object_no_row'] = prefix + str(uuid.uuid4())
s3tool('put', f['F1_orphan_object_no_row'])
psql("DELETE FROM attachments WHERE id = '%s'" % u3['id'])
f['F2_metadata_row_lost(U3)'] = u3_key
s3tool('delete', key_of(u2['id']))
f['F3_object_lost_of_used(U2)'] = key_of(u2['id'])
f4_id = str(uuid.uuid4())
f['F4_crash_after_put_before_ready'] = prefix + f4_id
insert_pending(f4_id, cid, alice['id'], f['F4_crash_after_put_before_ready'], '1 hour')
s3tool('put', f['F4_crash_after_put_before_ready'])
f5_id = str(uuid.uuid4())
f['F5_crash_before_put'] = prefix + f5_id
insert_pending(f5_id, cid, alice['id'], f['F5_crash_before_put'], '1 hour')
print('waiting %ds so that injected objects are older than grace=%s' % (a.wait, a.grace), flush=True)
time.sleep(a.wait)
# fresh state that must NOT be touched: an object being uploaded right now and its pending row
f7_id = str(uuid.uuid4())
f['F6_F7_fresh_upload_in_progress'] = prefix + f7_id
insert_pending(f7_id, cid, alice['id'], f['F6_F7_fresh_upload_in_progress'], '0 seconds')
s3tool('put', f['F6_F7_fresh_upload_in_progress'])
f8 = prefix + str(uuid.uuid4())
f['F8_fresh_orphan_object'] = f8
s3tool('put', f8)
steps = []
before = counts()
dry = reconcile('--grace', a.grace)
after_dry = counts()
steps.append({'step': 'dry-run', 'report': summary(dry), 'items': dry['items'], 'state_before': before,
'state_after': after_dry, 'dry_run_changed_nothing': before == after_dry})
check(before == after_dry, 'dry-run modified state')
mine = scoped(dry, prefix)
steps[-1]['this_experiment'] = mine
steps[-1]['other_findings_outside_experiment'] = scoped(dry, prefix, invert=True)
check(mine == {'orphan_object': 2, 'stale_pending': 2, 'missing_object': 1, 'orphan_object/skip': 1},
'detection mismatch: %s' % mine)
ap = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #1', 'report': summary(ap), 'this_experiment': scoped(ap, prefix),
'items': [i for i in ap['items'] if i.get('key', '').startswith(prefix)], 'state_after': counts()})
again = reconcile('--apply', '--grace', a.grace)
steps.append({'step': 'apply #2 (repeat)', 'report': summary(again), 'this_experiment': scoped(again, prefix),
'state_after': counts()})
check(again['fixed'] == 0 and again['errors'] == 0, 'second apply was not a no-op')
d1 = c.call('GET', '/api/v1/attachments/' + u1['id'] + '/content', actor=bob)
msgs = c.call('GET', '/api/v1/chats/' + cid + '/messages?limit=100', actor=bob, schema='MessagePage')['json']['items']
verify = {
'U1_used_healthy_download_sha_ok': hashlib.sha256(d1['bytes']).hexdigest() == u1['sha256'],
'U2_metadata_kept': c.call('GET', '/api/v1/attachments/' + u2['id'], actor=bob)['status'],
'U2_content_status': c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', 404, actor=bob)['status'],
'U2_marked_missing': psql("SELECT missing_at IS NOT NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't',
'message_still_references_U1_U2': [m for m in msgs if m['id'] == msg['id']][0]['attachment_ids'] == [u1['id'], u2['id']],
'F1_orphan_deleted': not exists(f['F1_orphan_object_no_row']),
'F2_orphan_of_lost_row_deleted': not exists(u3_key),
'F4_row_and_object_deleted': not exists(f['F4_crash_after_put_before_ready'])
and psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f4_id) == '0',
'F5_row_deleted': psql("SELECT count(*) FROM attachments WHERE id = '%s'" % f5_id) == '0',
'F7_fresh_pending_row_kept': psql("SELECT status FROM attachments WHERE id = '%s'" % f7_id) == 'pending',
'F7_fresh_object_kept': exists(f['F6_F7_fresh_upload_in_progress']),
'F8_fresh_orphan_kept': exists(f8),
}
for k, v in verify.items():
check(v is True or v in (200, 404), k)
# restore U2 from a "backup" (same bytes): the reconciler clears the missing mark
s3put_bytes(key_of(u2['id']), PNG)
rec = reconcile('--apply', '--grace', a.grace)
d2 = c.call('GET', '/api/v1/attachments/' + u2['id'] + '/content', actor=bob)
verify['U2_restored_and_unmarked'] = (rec['recovered'] == 1 and
hashlib.sha256(d2['bytes']).hexdigest() == u2['sha256'] and
psql("SELECT missing_at IS NULL FROM attachments WHERE id = '%s'" % u2['id']) == 't')
steps.append({'step': 'apply after restoring U2 object', 'report': summary(rec)})
check(verify['U2_restored_and_unmarked'], 'restore not detected')
# default production grace (15m): the same kind of fresh garbage is only reported as skipped
prod = reconcile('--grace', '15m')
steps.append({'step': 'dry-run with production grace 15m', 'report': summary(prod)})
# clean the fresh leftovers with the regular procedure so the next run starts clean
return {'grace_used': a.grace, 'faults': f, 'steps': steps, 'verify': verify}
def cmd_cost(a):
c = Client(a.base_url, timeout=30)
alice, bob, eve, chat = c.bootstrap()
run = uuid.uuid4().hex[:8]
keys_file = ROOT / 'evidence/track2/.bench-keys.txt'
created = 0
results = []
try:
for n in sorted(a.sizes):
add = n - created
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created, n)]
psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,"
" width, height, object_key, status, created_at, updated_at)"
" SELECT gen_random_uuid(), '%s', '%s', 'bench.png', 'image/png', 100, repeat('0', 64), 1, 1,"
" 'attachments/bench-%s/' || lpad(i::text, 7, '0'), 'ready', now() - interval '1 day', now()"
" FROM generate_series(%d, %d) AS i" % (chat['id'], alice['id'], run, created, n - 1))
t0 = time.monotonic()
put = json.loads(subprocess.run(
['docker', 'run', '--rm', '-i', '--network', NETWORK, '-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-put', '--size', '100', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True).stdout.strip().splitlines()[-1])
check(put['ok'] == add, 'bulk put incomplete: %s' % put)
created = n
print('populated %d (+%d in %.1fs)' % (n, add, time.monotonic() - t0), flush=True)
for r in range(a.repeat):
rep = reconcile('--grace', '15m')
row = {'bench_objects': n, 'repeat': r + 1, 'objects_scanned': rep['objects_scanned'],
'rows_scanned': rep['rows_scanned'], 'list_ms': rep['list_ms'], 'db_ms': rep['db_ms'],
'total_ms': rep['total_ms'], 'per_object_us': round(rep['per_object_us'], 1),
'container_memory_peak_mib': rep['container_memory_peak_mib'],
'findings': rep['orphan_objects'] + rep['stale_pending'] + rep['missing_objects']}
print(json.dumps(row), flush=True)
results.append(row)
finally:
if created:
keys = ['attachments/bench-%s/%07d' % (run, i) for i in range(created)]
psql("DELETE FROM attachments WHERE object_key LIKE 'attachments/bench-%s/%%'" % run)
subprocess.run(['docker', 'run', '--rm', '-i', '--network', NETWORK,
'-v', winpath(ROOT / 'secrets') + ':/secrets:ro',
'-v', winpath(ROOT / 'scripts/track2') + ':/tool:ro', TOOL_IMAGE, 'python', '-I',
'/tool/s3tool.py', 'bulk-delete', '--workers', '32'],
input='\n'.join(keys), capture_output=True, text=True)
if keys_file.exists():
keys_file.unlink()
return {'object_size_bytes': 100, 'reconciler_limits': '0.5 CPU, 128M', 's3': 'SeaweedFS 3.80, 1 CPU, 1G',
'runs': results}
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['faults', 'cost'])
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--grace', default='1m')
p.add_argument('--wait', type=int, default=65)
p.add_argument('--sizes', type=int, nargs='+', default=[1000, 5000, 20000])
p.add_argument('--repeat', type=int, default=3)
p.add_argument('--output')
a = p.parse_args()
sh('docker', 'compose', 'stop', 'reconciler')
try:
res = {'command': a.cmd, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
'commit': sh('git', 'rev-parse', '--short', 'HEAD', check=False),
'result': (cmd_faults if a.cmd == 'faults' else cmd_cost)(a)}
finally:
sh('docker', 'compose', 'start', 'reconciler')
text = json.dumps(res, ensure_ascii=False, indent=2)
if a.output:
Path(a.output).parent.mkdir(parents=True, exist_ok=True)
Path(a.output).write_text(text + '\n', encoding='utf-8')
print(text if a.cmd == 'faults' else json.dumps(res['result']['runs'][-1]))
print('PASS: track2 ' + a.cmd)
if __name__ == '__main__':
main()
+143
View File
@@ -0,0 +1,143 @@
#!/usr/bin/env python3
"""Minimal S3 client (stdlib, SigV4) for checks and fault injection OUTSIDE the API.
Runs inside a throwaway container on the compose `data` network (S3 is not
published to the host), see run_in_net() in lab4ops.py. Credentials are read
from mounted secret files and never printed.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import hmac
import http.client
import json
import os
import sys
import threading
from urllib.parse import quote
import xml.etree.ElementTree as ET
HOST = os.environ.get('S3_HOST', 's3:8333')
REGION = os.environ.get('S3_REGION', 'us-east-1')
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
SECRETS = os.environ.get('SECRETS_DIR', '/secrets')
NS = '{http://s3.amazonaws.com/doc/2006-03-01/}'
def creds(identity):
if identity == 'anonymous':
return None
prefix = 's3_admin_' if identity == 'admin' else 's3_'
read = lambda n: open(os.path.join(SECRETS, prefix + n)).read().strip()
return read('access_key_id'), read('secret_access_key')
def _h(key, msg):
return hmac.new(key, msg.encode(), hashlib.sha256).digest()
_local = threading.local()
def request(method, path, identity='api', body=b'', query=None):
query = query or {}
headers = {'host': HOST}
payload_hash = hashlib.sha256(body).hexdigest()
c = creds(identity)
if c:
now = dt.datetime.now(dt.timezone.utc)
amzdate, date = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
headers.update({'x-amz-content-sha256': payload_hash, 'x-amz-date': amzdate})
signed = ';'.join(sorted(headers))
cq = '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
creq = '\n'.join([method, quote(path, safe='/-_.~'), cq,
''.join('%s:%s\n' % (k, headers[k]) for k in sorted(headers)), signed, payload_hash])
scope = '%s/%s/s3/aws4_request' % (date, REGION)
sts = '\n'.join(['AWS4-HMAC-SHA256', amzdate, scope, hashlib.sha256(creq.encode()).hexdigest()])
k = _h(_h(_h(_h(('AWS4' + c[1]).encode(), date), REGION), 's3'), 'aws4_request')
sig = hmac.new(k, sts.encode(), hashlib.sha256).hexdigest()
headers['authorization'] = 'AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s' % (
c[0], scope, signed, sig)
url = quote(path, safe='/-_.~')
if query:
url += '?' + '&'.join('%s=%s' % (quote(k, safe='-_.~'), quote(v, safe='-_.~')) for k, v in sorted(query.items()))
conn = getattr(_local, 'conn', None)
if conn is None:
conn = _local.conn = http.client.HTTPConnection(HOST, timeout=30)
try:
conn.request(method, url, body=body or None, headers=headers)
r = conn.getresponse()
return r.status, r.read()
except (OSError, http.client.HTTPException):
_local.conn = None
raise
def obj(key):
return '/' + BUCKET + '/' + key
def list_keys(prefix, identity='api'):
keys, token = [], None
while True:
q = {'list-type': '2', 'prefix': prefix, 'max-keys': '1000'}
if token:
q['continuation-token'] = token
st, body = request('GET', '/' + BUCKET, identity, query=q)
if st != 200:
raise SystemExit('list failed: HTTP %d' % st)
root = ET.fromstring(body)
keys += [e.find(NS + 'Key').text for e in root.findall(NS + 'Contents')]
if root.findtext(NS + 'IsTruncated') != 'true':
return keys
token = root.findtext(NS + 'NextContinuationToken')
def main():
p = argparse.ArgumentParser()
p.add_argument('cmd', choices=['get', 'put', 'delete', 'create-bucket', 'list', 'count', 'bulk-put', 'bulk-delete'])
p.add_argument('arg', nargs='?', default='')
p.add_argument('--as', dest='identity', default='api', choices=['api', 'admin', 'anonymous'])
p.add_argument('--size', type=int, default=100)
p.add_argument('--workers', type=int, default=32)
p.add_argument('--stdin', action='store_true', help='put: body from stdin instead of random bytes')
a = p.parse_args()
out = {'cmd': a.cmd, 'as': a.identity}
if a.cmd == 'get':
st, body = request('GET', obj(a.arg), a.identity)
out.update(status=st, bytes=len(body) if st == 200 else 0,
sha256=hashlib.sha256(body).hexdigest() if st == 200 else None)
elif a.cmd == 'put':
body = sys.stdin.buffer.read() if a.stdin else os.urandom(a.size)
st, _ = request('PUT', obj(a.arg), a.identity, body=body)
out.update(status=st)
elif a.cmd == 'delete':
st, _ = request('DELETE', obj(a.arg), a.identity)
out.update(status=st)
elif a.cmd == 'create-bucket':
st, _ = request('PUT', '/' + a.arg, a.identity)
out.update(status=st)
elif a.cmd == 'list':
out.update(keys=list_keys(a.arg, a.identity))
elif a.cmd == 'count':
out.update(count=len(list_keys(a.arg, a.identity)))
elif a.cmd in ('bulk-put', 'bulk-delete'):
keys = [k.strip() for k in sys.stdin if k.strip()]
data = b'x' * a.size
def one(k):
if a.cmd == 'bulk-put':
return request('PUT', obj(k), a.identity, body=data)[0]
return request('DELETE', obj(k), a.identity)[0]
start = dt.datetime.now()
with ThreadPoolExecutor(a.workers) as pool:
codes = list(pool.map(one, keys))
out.update(count=len(keys), ok=sum(1 for c in codes if c in (200, 204)),
seconds=round((dt.datetime.now() - start).total_seconds(), 2))
print(json.dumps(out))
if __name__ == '__main__':
main()
+331
View File
@@ -0,0 +1,331 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"log/slog"
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"sync/atomic"
"syscall"
"time"
"mtusi-messenger/internal/api"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/cache"
"mtusi-messenger/internal/db"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/reconcile"
"mtusi-messenger/internal/store"
)
func getenv(key, def string) string {
if v := os.Getenv(key); v != "" {
return v
}
return def
}
func getint(key string, def int) int {
v := os.Getenv(key)
if v == "" {
return def
}
n, err := strconv.Atoi(v)
if err != nil || n <= 0 {
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
os.Exit(2)
}
return n
}
func secret(key string) (string, error) {
if path := os.Getenv(key + "_FILE"); path != "" {
b, err := os.ReadFile(path)
if err != nil {
return "", fmt.Errorf("read %s_FILE: %w", key, err)
}
return strings.TrimSpace(string(b)), nil
}
if v := os.Getenv(key); v != "" {
return v, nil
}
return "", fmt.Errorf("%s or %s_FILE is required", key, key)
}
func main() {
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
if err := run(log); err != nil {
log.Error("fatal", "error", err.Error())
os.Exit(1)
}
}
func openDB(ctx context.Context) (*store.Store, func(), error) {
dsn, err := secret("DATABASE_URL")
if err != nil {
return nil, nil, err
}
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: int32(getint("DB_MAX_CONNS", 10))})
if err != nil {
return nil, nil, errors.New("database config is invalid")
}
return store.New(pool), pool.Close, nil
}
func getduration(key string, def time.Duration) time.Duration {
v := os.Getenv(key)
if v == "" {
return def
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
fmt.Fprintf(os.Stderr, "invalid %s\n", key)
os.Exit(2)
}
return d
}
func openObjects() (*objstore.Store, error) { return openObjectsAs("S3") }
// openObjectsAs reads <prefix>_ACCESS_KEY_ID[_FILE] / <prefix>_SECRET_ACCESS_KEY[_FILE]:
// S3 for the restricted API identity, S3_ADMIN only in the one-shot bootstrap.
func openObjectsAs(prefix string) (*objstore.Store, error) {
access, err := secret(prefix + "_ACCESS_KEY_ID")
if err != nil {
return nil, err
}
secretKey, err := secret(prefix + "_SECRET_ACCESS_KEY")
if err != nil {
return nil, err
}
return objstore.New(objstore.Config{
Endpoint: getenv("S3_ENDPOINT", "http://s3:9000"),
Region: getenv("S3_REGION", "us-east-1"),
Bucket: getenv("S3_BUCKET", "messenger-attachments"),
AccessKey: access,
SecretKey: secretKey,
})
}
// runReconcile: `server reconcile [--apply] [--grace 15m] [--every 10m]`.
// Dry-run by default; prints one JSON report per pass.
func runReconcile(ctx context.Context, log *slog.Logger, args []string) error {
fs := flag.NewFlagSet("reconcile", flag.ContinueOnError)
apply := fs.Bool("apply", false, "apply fixes (default: dry-run, report only)")
grace := fs.Duration("grace", getduration("RECONCILE_GRACE", 15*time.Minute), "minimum age before an object/row is touched")
every := fs.Duration("every", 0, "repeat interval; 0 runs once")
if err := fs.Parse(args); err != nil {
return err
}
if *grace < time.Minute {
return errors.New("grace below 1m is unsafe: it is shorter than an upload may take")
}
st, closeDB, err := openDB(ctx)
if err != nil {
return err
}
defer closeDB()
objects, err := openObjects()
if err != nil {
return err
}
rec := reconcile.New(st, objects, log)
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
for {
pctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
rep, err := rec.Run(pctx, reconcile.Options{Apply: *apply, Grace: *grace})
cancel()
if err != nil {
if *every == 0 {
return err
}
log.Warn("reconcile pass failed", "error", err.Error())
} else {
out, _ := json.Marshal(rep)
fmt.Println(string(out))
}
if *every == 0 {
return nil
}
select {
case <-stop:
return nil
case <-time.After(*every):
}
}
}
func run(log *slog.Logger) error {
ctx := context.Background()
if len(os.Args) > 1 && os.Args[1] == "migrate" {
dsn, err := secret("DATABASE_URL")
if err != nil {
return err
}
pool, err := db.Open(ctx, dsn, db.PoolOptions{MaxConns: 2})
if err != nil {
return errors.New("database config is invalid")
}
defer pool.Close()
mctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
return db.Migrate(mctx, pool, log)
}
if len(os.Args) > 1 && os.Args[1] == "s3-init" {
objects, err := openObjectsAs("S3_ADMIN")
if err != nil {
return err
}
var created bool
for attempt := 1; ; attempt++ {
bctx, cancel := context.WithTimeout(ctx, 5*time.Second)
created, err = objects.EnsureBucket(bctx, getenv("S3_REGION", "us-east-1"))
cancel()
if err == nil || attempt == 30 {
break
}
log.Warn("s3 not ready, retrying", "attempt", attempt, "error", err.Error())
time.Sleep(2 * time.Second)
}
if err != nil {
return err
}
log.Info("bucket ready", "bucket", objects.Bucket(), "created", created)
return nil
}
if len(os.Args) > 1 && os.Args[1] == "reconcile" {
return runReconcile(ctx, log, os.Args[2:])
}
st, closeDB, err := openDB(ctx)
if err != nil {
return err
}
defer closeDB()
ttl := time.Duration(getint("SESSION_TTL_SECONDS", 3600)) * time.Second
cacheMaxTTL := time.Duration(getint("SESSION_CACHE_MAX_TTL_SECONDS", 60)) * time.Second
var sessCache auth.SessionCache
switch mode := getenv("SESSION_STORE", "postgres"); mode {
case "postgres":
case "valkey", "redis":
url, err := secret("CACHE_URL")
if err != nil {
return err
}
vk, err := cache.NewValkey(url)
if err != nil {
return errors.New("cache config is invalid")
}
defer vk.Close()
sessCache = vk
default:
return fmt.Errorf("unknown SESSION_STORE %q (postgres|valkey|redis)", mode)
}
hasher := auth.NewHasher(getint("PASSWORD_HASH_CONCURRENCY", 4))
sessions := auth.NewSessions(st, sessCache, ttl, cacheMaxTTL, log)
instanceID := getenv("INSTANCE_ID", "")
if instanceID == "" {
h, err := os.Hostname()
if err != nil || h == "" {
return errors.New("cannot determine instance id; set INSTANCE_ID")
}
instanceID = h
}
idemTTL := time.Duration(getint("IDEMPOTENCY_TTL_SECONDS", 86400)) * time.Second
objects, err := openObjects()
if err != nil {
return err
}
maxUpload := int64(getint("MAX_UPLOAD_BYTES", 10485760))
if maxUpload > 10485760 {
return errors.New("MAX_UPLOAD_BYTES above the 10 MiB contract limit")
}
drainDelay := time.Duration(getint("SHUTDOWN_DRAIN_MS", 3000)) * time.Millisecond
ready := &atomic.Bool{}
ready.Store(true)
host := getenv("APP_HOST", "0.0.0.0")
port := getenv("APP_PORT", "8080")
addr := host + ":" + port
srv := api.NewServer(st, objects, sessions, hasher, log, ready, api.Options{
InstanceID: instanceID, IdempotencyTTL: idemTTL,
MaxUploadBytes: maxUpload,
MaxImagePixels: int64(getint("MAX_IMAGE_PIXELS", 40000000)),
UploadConcurrency: int64(getint("UPLOAD_CONCURRENCY", 8)),
DecodeBudgetBytes: int64(getint("DECODE_BUDGET_BYTES", 128<<20)),
UploadTimeout: getduration("UPLOAD_TIMEOUT", 60*time.Second),
S3Timeout: getduration("S3_TIMEOUT", 15*time.Second),
})
httpSrv := &http.Server{
Addr: addr,
Handler: srv.Handler(),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 90 * time.Second,
WriteTimeout: 90 * time.Second,
IdleTimeout: 60 * time.Second,
}
bgCtx, stopBg := context.WithCancel(ctx)
defer stopBg()
go func() {
t := time.NewTicker(10 * time.Minute)
defer t.Stop()
for {
select {
case <-bgCtx.Done():
return
case <-t.C:
c, cancel := context.WithTimeout(bgCtx, 10*time.Second)
if n, err := st.PurgeSessions(c, 24*time.Hour); err == nil && n > 0 {
log.Info("expired sessions purged", "count", n)
}
if n, err := st.PurgeIdempotency(c, idemTTL); err == nil && n > 0 {
log.Info("expired idempotency keys purged", "count", n)
}
cancel()
}
}
}()
errCh := make(chan error, 1)
go func() { errCh <- httpSrv.ListenAndServe() }()
log.Info("server started", "addr", addr, "pid", os.Getpid(), "instance_id", instanceID, "build", getenv("APP_BUILD", "dev"), "s3_bucket", objects.Bucket(),
"session_store", getenv("SESSION_STORE", "postgres"), "session_ttl_s", int(ttl.Seconds()))
stop := make(chan os.Signal, 1)
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
select {
case err := <-errCh:
if err != nil && !errors.Is(err, http.ErrServerClosed) {
return err
}
case sig := <-stop:
log.Info("shutdown signal received", "signal", sig.String())
ready.Store(false)
time.Sleep(drainDelay)
httpSrv.SetKeepAlivesEnabled(false)
sctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
if err := httpSrv.Shutdown(sctx); err != nil {
log.Warn("graceful shutdown exceeded budget, closing", "error", err.Error())
_ = httpSrv.Close()
}
log.Info("server stopped cleanly")
}
return nil
}
+30
View File
@@ -0,0 +1,30 @@
module mtusi-messenger
go 1.23
require (
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.7.2
github.com/minio/minio-go/v7 v7.0.82
github.com/redis/go-redis/v9 v9.7.0
golang.org/x/crypto v0.31.0
golang.org/x/sync v0.10.0
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/puddle/v2 v2.2.2 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/rs/xid v1.6.0 // indirect
golang.org/x/net v0.30.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
)
+62
View File
@@ -0,0 +1,62 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
github.com/jackc/pgx/v5 v5.7.2 h1:mLoDLV6sonKlvjIEsV56SkWNCnuNv531l94GaIzO+XI=
github.com/jackc/pgx/v5 v5.7.2/go.mod h1:ncY89UGWxg82EykZUwSpUKEfccBGGYq1xjrOpsbsfGQ=
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.82 h1:tWfICLhmp2aFPXL8Tli0XDTHj2VB/fNf0PC1f/i1gRo=
github.com/minio/minio-go/v7 v7.0.82/go.mod h1:84gmIilaX4zcvAWWzJ5Z1WI5axN+hAbM5w25xf8xvC0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+338
View File
@@ -0,0 +1,338 @@
package api
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
"image/color"
"image/jpeg"
"image/png"
"io"
"mime"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/store"
)
const (
mimePNG = "image/png"
mimeJPEG = "image/jpeg"
// An oversized body is read and discarded up to this many bytes so the client
// receives 413 instead of a connection reset; beyond it the connection is closed.
maxDrainBytes = 64 << 20
)
func (s *Server) objectError(w http.ResponseWriter, r *http.Request, err error) {
s.log.Warn("object storage unavailable", "request_id", requestID(r), "error", err.Error())
apperr.Write(w, requestID(r), apperr.Unavailable("attachment storage temporarily unavailable"))
}
func validFilename(name string) bool {
if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || name == "." || name == ".." {
return false
}
if n := runeLen(name); n < 1 || n > 255 {
return false
}
for _, c := range name {
switch {
case c == '/', c == '\\', c == 0, unicode.IsControl(c):
return false
case c >= 0x202a && c <= 0x202e, c >= 0x2066 && c <= 0x2069: // bidi overrides hide the real extension
return false
}
}
return true
}
func sniffImage(data []byte) string {
switch {
case bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")):
return mimePNG
case bytes.HasPrefix(data, []byte{0xff, 0xd8, 0xff}):
return mimeJPEG
}
return ""
}
// decodeCost estimates bytes allocated by a full decode for the memory budget.
func decodeCost(m color.Model, pixels int64) int64 {
bpp := int64(4)
switch m {
case color.GrayModel, color.AlphaModel:
bpp = 1
case color.Gray16Model, color.Alpha16Model:
bpp = 2
case color.YCbCrModel:
bpp = 3
case color.RGBA64Model, color.NRGBA64Model:
bpp = 8
}
if _, ok := m.(color.Palette); ok {
bpp = 1
}
return pixels * bpp
}
type imageInfo struct {
contentType string
width, height int
}
// inspectImage checks the real format by magic bytes, compares it with the
// declared Content-Type, enforces the pixel limit from the header only and then
// fully decodes the image under a shared memory budget to reject corrupt data.
func (s *Server) inspectImage(ctx context.Context, data []byte, declared string) (imageInfo, *apperr.AppError) {
detected := sniffImage(data)
if detected == "" {
return imageInfo{}, apperr.UnsupportedMedia("content is not a PNG or JPEG image")
}
if detected != declared {
return imageInfo{}, apperr.UnsupportedMedia("Content-Type does not match the image content")
}
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || cfg.Width <= 0 || cfg.Height <= 0 {
return imageInfo{}, apperr.UnsupportedMedia("image header is invalid")
}
pixels := int64(cfg.Width) * int64(cfg.Height)
if pixels > s.maxPixels {
return imageInfo{}, apperr.PayloadTooLarge("image exceeds " + strconv.FormatInt(s.maxPixels, 10) + " pixels")
}
cost := min(decodeCost(cfg.ColorModel, pixels), s.decodeBudget)
if err := s.decodeSem.Acquire(ctx, cost); err != nil {
return imageInfo{}, apperr.Unavailable("server is busy processing images, retry later")
}
if detected == mimePNG {
_, err = png.Decode(bytes.NewReader(data))
} else {
_, err = jpeg.Decode(bytes.NewReader(data))
}
s.decodeSem.Release(cost)
if err != nil {
return imageInfo{}, apperr.UnsupportedMedia("image data is corrupted")
}
return imageInfo{contentType: detected, width: cfg.Width, height: cfg.Height}, nil
}
func readBody(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, *apperr.AppError) {
var buf bytes.Buffer
if r.ContentLength > 0 {
buf.Grow(int(r.ContentLength))
}
_, err := buf.ReadFrom(http.MaxBytesReader(w, r.Body, limit))
var maxErr *http.MaxBytesError
switch {
case errors.As(err, &maxErr):
return nil, apperr.PayloadTooLarge("attachment exceeds " + strconv.FormatInt(limit, 10) + " bytes")
case err != nil:
return nil, apperr.InvalidRequest("request body could not be read")
case buf.Len() == 0:
return nil, apperr.InvalidRequest("request body is empty")
}
return buf.Bytes(), nil
}
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
body := r.Body
defer func() { _, _ = io.CopyN(io.Discard, body, maxDrainBytes) }()
id := requestID(r)
chatID, ok := s.chatAccess(w, r)
if !ok {
return
}
query, err := url.ParseQuery(r.URL.RawQuery)
if err != nil || len(query["filename"]) != 1 {
apperr.Write(w, id, apperr.InvalidRequest("exactly one filename query parameter is required"))
return
}
filename := query.Get("filename")
if !validFilename(filename) {
apperr.Write(w, id, apperr.InvalidRequest("filename must be 1..255 code points without path separators or control characters"))
return
}
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if media != mimePNG && media != mimeJPEG {
apperr.Write(w, id, apperr.UnsupportedMedia("Content-Type must be image/png or image/jpeg"))
return
}
if r.ContentLength > s.maxUpload {
apperr.Write(w, id, apperr.PayloadTooLarge("attachment exceeds "+strconv.FormatInt(s.maxUpload, 10)+" bytes"))
return
}
// Each slot may hold up to maxUpload bytes in memory; waiting is bounded by the request timeout.
if err := s.uploadSem.Acquire(r.Context(), 1); err != nil {
apperr.Write(w, id, apperr.Unavailable("too many concurrent uploads, retry later"))
return
}
defer s.uploadSem.Release(1)
data, aerr := readBody(w, r, s.maxUpload)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
info, aerr := s.inspectImage(r.Context(), data, media)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
sum := sha256.Sum256(data)
attID := uuid.NewString()
att := &model.Attachment{
ID: attID, ChatID: chatID, OwnerID: currentUser(r).ID, Filename: filename,
ContentType: info.contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]),
Status: "ready", Variants: []model.Variant{},
// The key is server-generated; filename never reaches S3.
ObjectKey: "attachments/" + chatID + "/" + attID,
}
// 1. intent row (pending) -> 2. S3 PUT -> 3. pending->ready. A crash between
// the steps leaves a pending row and maybe an object; both are removed by the
// reconciler after the grace period, never an untracked object.
if err := s.store.InsertPendingAttachment(r.Context(), att, info.width, info.height); err != nil {
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
s.storeError(w, r, err)
return
}
putCtx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
err = s.objects.Put(putCtx, att.ObjectKey, data, att.ContentType)
cancel()
if err != nil {
go s.compensateUpload(att, id)
s.objectError(w, r, err)
return
}
updated, err := s.store.MarkAttachmentReady(r.Context(), att.ID)
if err != nil {
go s.compensateUpload(att, id)
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.Unavailable("upload expired, retry"))
return
}
s.storeError(w, r, err)
return
}
att.UpdatedAt = model.NewTime(updated)
writeJSON(w, r, http.StatusCreated, att)
}
// compensateUpload is best effort and starts from the row: the object is removed
// only if this call deleted the still-pending row. If the row is already ready
// (the UPDATE committed but its reply was lost) the object is kept. Anything left
// behind (pending row while PostgreSQL is down, or an object whose removal
// failed) is cleaned up by the reconciler after the grace period.
func (s *Server) compensateUpload(att *model.Attachment, reqID string) {
dctx, dcancel := context.WithTimeout(context.Background(), 3*time.Second)
defer dcancel()
deleted, err := s.store.DeletePendingAttachment(dctx, att.ID)
if err != nil || !deleted {
if err != nil {
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
"attachment_id", att.ID, "stage", "delete_row")
}
return
}
octx, ocancel := context.WithTimeout(context.Background(), 3*time.Second)
defer ocancel()
if err := s.objects.Remove(octx, att.ObjectKey); err != nil {
s.log.Warn("upload compensation deferred to reconciler", "request_id", reqID,
"attachment_id", att.ID, "stage", "remove_object")
}
}
func (s *Server) attachmentForRequest(w http.ResponseWriter, r *http.Request) (*model.Attachment, bool) {
id := requestID(r)
attID, err := parseUUID(r.PathValue("attachment_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("attachment_id must be a UUID"))
return nil, false
}
att, err := s.store.AttachmentForUser(r.Context(), attID, currentUser(r).ID)
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("attachment not found or not accessible"))
return nil, false
}
if err != nil {
s.storeError(w, r, err)
return nil, false
}
return att, true
}
func (s *Server) getAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
if att, ok := s.attachmentForRequest(w, r); ok {
writeJSON(w, r, http.StatusOK, att)
}
}
func (s *Server) downloadAttachment(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
id := requestID(r)
att, ok := s.attachmentForRequest(w, r)
if !ok {
return
}
switch v := r.URL.Query().Get("variant"); v {
case "", "original":
case "thumbnail", "crop", "optimized":
apperr.Write(w, id, apperr.NotFound("variant not found"))
return
default:
apperr.Write(w, id, apperr.InvalidRequest("unknown variant"))
return
}
ctx, cancel := context.WithTimeout(r.Context(), s.s3Timeout)
defer cancel()
obj, size, err := s.objects.Get(ctx, att.ObjectKey)
if objstore.IsNotFound(err) {
s.log.Error("attachment object missing in S3", "request_id", id, "attachment_id", att.ID)
apperr.Write(w, id, apperr.NotFound("attachment content is not available"))
return
}
if err != nil {
s.objectError(w, r, err)
return
}
defer obj.Close()
h := w.Header()
h.Set("Content-Type", att.ContentType)
h.Set("Content-Length", strconv.FormatInt(size, 10))
h.Set("X-Content-Type-Options", "nosniff")
h.Set("Cache-Control", "private, no-store")
h.Set("Content-Disposition", "inline; filename*=UTF-8''"+url.PathEscape(att.Filename))
h.Set("ETag", `"`+att.SHA256+`"`)
w.WriteHeader(http.StatusOK)
if _, err := io.Copy(w, obj); err != nil {
s.log.Warn("download interrupted", "request_id", id, "attachment_id", att.ID, "error", err.Error())
}
}
+427
View File
@@ -0,0 +1,427 @@
package api
import (
"context"
"encoding/base64"
"errors"
"net/http"
"strings"
"time"
"unicode/utf8"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/store"
)
func (s *Server) storeError(w http.ResponseWriter, r *http.Request, err error) {
id := requestID(r)
if store.IsUnavailable(err) || errors.Is(err, auth.ErrCacheUnavailable) {
s.log.Warn("dependency unavailable", "request_id", id, "error", err.Error())
apperr.Write(w, id, apperr.Unavailable("storage temporarily unavailable"))
return
}
s.log.Error("unexpected storage error", "request_id", id, "error", err.Error())
apperr.Write(w, id, apperr.Internal("internal error"))
}
func (s *Server) healthLive(w http.ResponseWriter, r *http.Request) {
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
}
func (s *Server) healthReady(w http.ResponseWriter, r *http.Request) {
if !s.ready.Load() {
apperr.Write(w, requestID(r), apperr.Unavailable("service is shutting down"))
return
}
ctx, cancel := context.WithTimeout(r.Context(), time.Second)
defer cancel()
if err := s.store.Ping(ctx); err != nil {
apperr.Write(w, requestID(r), apperr.Unavailable("database is not reachable"))
return
}
writeJSON(w, r, http.StatusOK, model.Health{Status: "ok"})
}
type createUserReq struct {
Username string `json:"username"`
DisplayName string `json:"display_name"`
Password string `json:"password"`
}
func validPassword(p string) bool {
n := utf8.RuneCountInString(p)
return utf8.ValidString(p) && n >= 12 && n <= 128
}
func (s *Server) usersRoute(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
id := requestID(r)
var req createUserReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
apperr.Write(w, id, aerr)
return
}
if !usernameRe.MatchString(req.Username) {
apperr.Write(w, id, apperr.InvalidRequest("username must match ^[a-z0-9_]{3,32}$"))
return
}
if !validText(req.DisplayName, 100) {
apperr.Write(w, id, apperr.InvalidRequest("display_name must be 1..100 non-space code points"))
return
}
if !validPassword(req.Password) {
apperr.Write(w, id, apperr.InvalidRequest("password must be 12..128 code points"))
return
}
hash, err := s.hasher.Hash(r.Context(), req.Password)
if err != nil {
s.storeError(w, r, err)
return
}
u, err := s.store.CreateUser(r.Context(), req.Username, req.DisplayName, hash)
if errors.Is(err, store.ErrUsernameTaken) {
apperr.Write(w, id, apperr.Conflict("username is already taken"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, u)
}
func (s *Server) me(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
writeJSON(w, r, http.StatusOK, currentUser(r))
}
const basicChallenge = `Basic realm="messenger", charset="UTF-8"`
func unauthorizedBasic(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", basicChallenge)
apperr.Write(w, requestID(r), apperr.Unauthorized("invalid username or password"))
}
func parseBasic(r *http.Request) (string, string, bool) {
scheme, value, ok := strings.Cut(r.Header.Get("Authorization"), " ")
if !ok || !strings.EqualFold(scheme, "Basic") {
return "", "", false
}
raw, err := base64.StdEncoding.DecodeString(strings.TrimSpace(value))
if err != nil || !utf8.Valid(raw) {
return "", "", false
}
user, pass, ok := strings.Cut(string(raw), ":")
if !ok || !usernameRe.MatchString(user) || !validPassword(pass) {
return "", "", false
}
return user, pass, true
}
func (s *Server) createSession(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
methodNotAllowed(w, r)
return
}
username, password, ok := parseBasic(r)
if !ok {
unauthorizedBasic(w, r)
return
}
user, hash, err := s.store.Credentials(r.Context(), username)
if err != nil && !errors.Is(err, store.ErrNotFound) {
s.storeError(w, r, err)
return
}
valid, err := s.hasher.Verify(r.Context(), password, hash)
if err != nil {
s.storeError(w, r, err)
return
}
if !valid {
unauthorizedBasic(w, r)
return
}
token, expires, err := s.sessions.Create(r.Context(), user)
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, model.Session{
Token: token, TokenType: "Bearer", ExpiresAt: model.NewTime(expires), User: *user,
})
}
func (s *Server) revokeSession(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodDelete {
methodNotAllowed(w, r)
return
}
err := s.sessions.Revoke(r.Context(), currentToken(r))
if errors.Is(err, auth.ErrInvalidSession) {
unauthorizedBearer(w, r)
return
}
if err != nil {
s.storeError(w, r, err)
return
}
w.Header().Set("X-Request-Id", requestID(r))
w.WriteHeader(http.StatusNoContent)
}
type createChatReq struct {
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
}
func (s *Server) chatsRoute(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
s.createChat(w, r)
case http.MethodGet:
s.listChats(w, r)
default:
methodNotAllowed(w, r)
}
}
func (s *Server) createChat(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
var req createChatReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
apperr.Write(w, id, aerr)
return
}
if !validText(req.Title, 100) {
apperr.Write(w, id, apperr.InvalidRequest("title must be 1..100 non-space code points"))
return
}
if len(req.MemberIDs) < 1 || len(req.MemberIDs) > 100 {
apperr.Write(w, id, apperr.InvalidRequest("member_ids must contain 1..100 items"))
return
}
seen := make(map[string]bool, len(req.MemberIDs))
for _, raw := range req.MemberIDs {
uid, err := parseUUID(raw)
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("member_ids must be UUID strings"))
return
}
if seen[uid] {
apperr.Write(w, id, apperr.InvalidRequest("duplicate member_id"))
return
}
seen[uid] = true
}
caller := currentUser(r)
if !seen[caller.ID] {
apperr.Write(w, id, apperr.InvalidRequest("creator must be included in member_ids"))
return
}
chat, err := s.store.CreateChat(r.Context(), caller.ID, req.Title, req.MemberIDs)
if errors.Is(err, store.ErrUnknownMember) {
apperr.Write(w, id, apperr.InvalidRequest("unknown member_id"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusCreated, chat)
}
func (s *Server) listChats(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
caller := currentUser(r)
limit, cur, aerr := pagingParams(r, caller.ID)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
items, more, err := s.store.ListChats(r.Context(), caller.ID, limit, cur)
if err != nil {
s.storeError(w, r, err)
return
}
page := model.Page[*model.Chat]{Items: items}
if more && len(items) > 0 {
last := items[len(items)-1]
encoded := store.EncodeCursor(store.Cursor{Scope: caller.ID, TS: last.CreatedAt.Time, ID: last.ID})
page.NextCursor = &encoded
}
writeJSON(w, r, http.StatusOK, page)
}
func (s *Server) chatRoute(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, r)
return
}
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return
}
chat, err := s.store.ChatForUser(r.Context(), chatID, currentUser(r).ID)
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
writeJSON(w, r, http.StatusOK, chat)
}
type createMessageReq struct {
Text string `json:"text"`
AttachmentIDs []string `json:"attachment_ids"`
}
func (s *Server) messagesRoute(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
s.createMessage(w, r)
case http.MethodGet:
s.listMessages(w, r)
default:
methodNotAllowed(w, r)
}
}
func (s *Server) chatAccess(w http.ResponseWriter, r *http.Request) (string, bool) {
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return "", false
}
ok, err := s.store.IsMember(r.Context(), chatID, currentUser(r).ID)
if err != nil {
s.storeError(w, r, err)
return "", false
}
if !ok {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return "", false
}
return chatID, true
}
func (s *Server) createMessage(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
chatID, err := parseUUID(r.PathValue("chat_id"))
if err != nil {
apperr.Write(w, id, apperr.InvalidRequest("chat_id must be a UUID"))
return
}
// Validation errors are reported only to members; outsiders always get 404.
reject := func(aerr *apperr.AppError) {
if _, ok := s.chatAccess(w, r); !ok {
return
}
apperr.Write(w, id, aerr)
}
key := r.Header.Get("Idempotency-Key")
if !idempotencyKeyRe.MatchString(key) {
reject(apperr.InvalidRequest("Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}"))
return
}
var req createMessageReq
if aerr := decodeJSON(w, r, &req); aerr != nil {
reject(aerr)
return
}
if req.AttachmentIDs == nil {
req.AttachmentIDs = []string{}
}
if len(req.AttachmentIDs) > 10 {
reject(apperr.InvalidRequest("attachment_ids must contain at most 10 items"))
return
}
seen := make(map[string]bool, len(req.AttachmentIDs))
for _, raw := range req.AttachmentIDs {
aid, err := parseUUID(raw)
if err != nil {
reject(apperr.InvalidRequest("attachment_ids must be UUID strings"))
return
}
if seen[aid] {
reject(apperr.InvalidRequest("duplicate attachment_id"))
return
}
seen[aid] = true
}
if len(req.AttachmentIDs) == 0 && !validText(req.Text, 4000) {
reject(apperr.InvalidRequest("text must be 1..4000 non-space code points when there are no attachments"))
return
}
if !utf8.ValidString(req.Text) || runeLen(req.Text) > 4000 {
reject(apperr.InvalidRequest("text must be at most 4000 code points"))
return
}
msg, replayed, err := s.store.AddMessage(r.Context(), chatID, currentUser(r).ID, req.Text, req.AttachmentIDs,
key, messageFingerprint(req), s.idempotencyTTL)
if errors.Is(err, store.ErrIdempotencyConflict) {
apperr.Write(w, id, apperr.Conflict("Idempotency-Key was already used with a different request"))
return
}
if errors.Is(err, store.ErrNotFound) {
apperr.Write(w, id, apperr.NotFound("chat not found or not accessible"))
return
}
if errors.Is(err, store.ErrAttachmentNotFound) {
apperr.Write(w, id, apperr.NotFound("attachment not found in this chat"))
return
}
if err != nil {
s.storeError(w, r, err)
return
}
if replayed {
w.Header().Set("Idempotent-Replayed", "true")
}
writeJSON(w, r, http.StatusCreated, msg)
}
func (s *Server) listMessages(w http.ResponseWriter, r *http.Request) {
id := requestID(r)
chatID, ok := s.chatAccess(w, r)
if !ok {
return
}
limit, cur, aerr := pagingParams(r, chatID)
if aerr != nil {
apperr.Write(w, id, aerr)
return
}
items, more, err := s.store.ListMessages(r.Context(), chatID, limit, cur)
if err != nil {
s.storeError(w, r, err)
return
}
page := model.Page[*model.Message]{Items: items}
if more && len(items) > 0 {
last := items[len(items)-1]
encoded := store.EncodeCursor(store.Cursor{Scope: chatID, TS: last.CreatedAt.Time, ID: last.ID})
page.NextCursor = &encoded
}
writeJSON(w, r, http.StatusOK, page)
}
func (s *Server) notFound(w http.ResponseWriter, r *http.Request) {
apperr.Write(w, requestID(r), apperr.NotFound("route not found"))
}
+110
View File
@@ -0,0 +1,110 @@
package api
import (
"crypto/sha256"
"encoding/json"
"errors"
"io"
"mime"
"net/http"
"regexp"
"strconv"
"strings"
"unicode/utf8"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/store"
)
const maxBodyBytes = 1 << 20
var (
usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
)
func messageFingerprint(req createMessageReq) []byte {
b, _ := json.Marshal(req)
sum := sha256.Sum256(b)
return sum[:]
}
func parseUUID(s string) (string, error) {
u, err := uuid.Parse(s)
if err != nil || u.String() != s {
return "", errors.New("invalid uuid")
}
return u.String(), nil
}
func runeLen(s string) int { return utf8.RuneCountInString(s) }
func validText(s string, max int) bool {
n := runeLen(s)
return n >= 1 && n <= max && strings.TrimSpace(s) != ""
}
func decodeJSON(w http.ResponseWriter, r *http.Request, dst any) *apperr.AppError {
media, _, _ := mime.ParseMediaType(r.Header.Get("Content-Type"))
if media != "application/json" {
return apperr.UnsupportedMedia("Content-Type must be application/json")
}
r.Body = http.MaxBytesReader(w, r.Body, maxBodyBytes)
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
if err := dec.Decode(dst); err != nil {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
return apperr.PayloadTooLarge("request body too large")
}
return apperr.InvalidRequest("malformed JSON body")
}
var extra json.RawMessage
if err := dec.Decode(&extra); err != io.EOF {
var maxErr *http.MaxBytesError
if errors.As(err, &maxErr) {
return apperr.PayloadTooLarge("request body too large")
}
return apperr.InvalidRequest("malformed JSON body")
}
return nil
}
func pagingParams(r *http.Request, scope string) (int, *store.Cursor, *apperr.AppError) {
limit := 50
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 1 || n > 100 {
return 0, nil, apperr.InvalidRequest("limit must be an integer in 1..100")
}
limit = n
}
var cur *store.Cursor
if v := r.URL.Query().Get("cursor"); v != "" {
c, err := store.DecodeCursor(v)
if err == nil {
_, err = parseUUID(c.ID)
}
if err != nil {
return 0, nil, apperr.InvalidRequest("invalid cursor")
}
if c.Scope != scope {
return 0, nil, apperr.InvalidRequest("cursor does not belong to this resource")
}
cur = &c
}
return limit, cur, nil
}
func writeJSON(w http.ResponseWriter, r *http.Request, status int, payload any) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("X-Request-Id", requestID(r))
w.WriteHeader(status)
_ = json.NewEncoder(w).Encode(payload)
}
func methodNotAllowed(w http.ResponseWriter, r *http.Request) {
apperr.Write(w, requestID(r), apperr.New(http.StatusMethodNotAllowed, apperr.CodeInvalidRequest, "method not allowed"))
}
+150
View File
@@ -0,0 +1,150 @@
package api
import (
"context"
"errors"
"log/slog"
"net/http"
"strings"
"time"
"github.com/google/uuid"
"mtusi-messenger/internal/apperr"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/model"
)
type ctxKey int
const (
ctxRequestID ctxKey = iota
ctxUser
ctxToken
)
func requestIDMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := r.Header.Get("X-Request-Id")
if id == "" {
id = uuid.NewString()
}
w.Header().Set("X-Request-Id", id)
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), ctxRequestID, id)))
})
}
func instanceIDMiddleware(instanceID string, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Instance-Id", instanceID)
next.ServeHTTP(w, r)
})
}
func requestID(r *http.Request) string {
if id, ok := r.Context().Value(ctxRequestID).(string); ok && id != "" {
return id
}
return "unknown"
}
type statusWriter struct {
http.ResponseWriter
status int
}
func (s *statusWriter) WriteHeader(code int) {
s.status = code
s.ResponseWriter.WriteHeader(code)
}
func loggingMiddleware(log *slog.Logger, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
sw := &statusWriter{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(sw, r)
log.Info("http",
"method", r.Method,
"path", r.URL.Path,
"status", sw.status,
"dur_ms", time.Since(start).Milliseconds(),
"request_id", requestID(r),
)
})
}
func recoverMiddleware(log *slog.Logger, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
if rec := recover(); rec != nil {
log.Error("panic recovered", "path", r.URL.Path, "request_id", requestID(r))
apperr.Write(w, requestID(r), apperr.Internal("internal error"))
}
}()
next.ServeHTTP(w, r)
})
}
// Binary transfers get a longer budget than JSON requests.
func timeoutMiddleware(d, binary time.Duration, next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
limit := d
if strings.Contains(r.URL.Path, "/attachments") {
limit = binary
}
ctx, cancel := context.WithTimeout(r.Context(), limit)
defer cancel()
next.ServeHTTP(w, r.WithContext(ctx))
})
}
const bearerChallenge = `Bearer realm="messenger"`
func unauthorizedBearer(w http.ResponseWriter, r *http.Request) {
w.Header().Set("WWW-Authenticate", bearerChallenge)
apperr.Write(w, requestID(r), apperr.Unauthorized("valid Bearer session token is required"))
}
func bearerToken(r *http.Request) (string, bool) {
scheme, token, ok := strings.Cut(r.Header.Get("Authorization"), " ")
if !ok || !strings.EqualFold(scheme, "Bearer") {
return "", false
}
token = strings.TrimSpace(token)
if token == "" || len(token) > 256 {
return "", false
}
return token, true
}
func (s *Server) auth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
token, ok := bearerToken(r)
if !ok {
unauthorizedBearer(w, r)
return
}
user, err := s.sessions.Resolve(r.Context(), token)
if errors.Is(err, auth.ErrInvalidSession) {
unauthorizedBearer(w, r)
return
}
if err != nil {
s.storeError(w, r, err)
return
}
ctx := context.WithValue(r.Context(), ctxUser, user)
ctx = context.WithValue(ctx, ctxToken, token)
next.ServeHTTP(w, r.WithContext(ctx))
}
}
func currentUser(r *http.Request) *model.User {
return r.Context().Value(ctxUser).(*model.User)
}
func currentToken(r *http.Request) string {
return r.Context().Value(ctxToken).(string)
}
+80
View File
@@ -0,0 +1,80 @@
package api
import (
"log/slog"
"net/http"
"sync/atomic"
"time"
"golang.org/x/sync/semaphore"
"mtusi-messenger/internal/auth"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/store"
)
type Server struct {
store *store.Store
objects *objstore.Store
sessions *auth.Sessions
hasher *auth.Hasher
log *slog.Logger
ready *atomic.Bool
timeout time.Duration
instanceID string
idempotencyTTL time.Duration
uploadTimeout time.Duration
s3Timeout time.Duration
maxUpload int64
maxPixels int64
decodeBudget int64
uploadSem *semaphore.Weighted
decodeSem *semaphore.Weighted
}
type Options struct {
InstanceID string
IdempotencyTTL time.Duration
MaxUploadBytes int64
MaxImagePixels int64
UploadConcurrency int64
DecodeBudgetBytes int64
UploadTimeout time.Duration
S3Timeout time.Duration
}
func NewServer(st *store.Store, objects *objstore.Store, sessions *auth.Sessions, hasher *auth.Hasher, log *slog.Logger, ready *atomic.Bool, opt Options) *Server {
return &Server{store: st, objects: objects, sessions: sessions, hasher: hasher, log: log, ready: ready, timeout: 8 * time.Second,
instanceID: opt.InstanceID, idempotencyTTL: opt.IdempotencyTTL,
uploadTimeout: opt.UploadTimeout, s3Timeout: opt.S3Timeout,
maxUpload: opt.MaxUploadBytes, maxPixels: opt.MaxImagePixels, decodeBudget: opt.DecodeBudgetBytes,
uploadSem: semaphore.NewWeighted(opt.UploadConcurrency), decodeSem: semaphore.NewWeighted(opt.DecodeBudgetBytes)}
}
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.HandleFunc("/health/live", s.healthLive)
mux.HandleFunc("/health/ready", s.healthReady)
mux.HandleFunc("/api/v1/users", s.usersRoute)
mux.HandleFunc("/api/v1/auth/sessions", s.createSession)
mux.HandleFunc("/api/v1/auth/sessions/current", s.auth(s.revokeSession))
mux.HandleFunc("/api/v1/users/me", s.auth(s.me))
mux.HandleFunc("/api/v1/chats", s.auth(s.chatsRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}", s.auth(s.chatRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}/messages", s.auth(s.messagesRoute))
mux.HandleFunc("/api/v1/chats/{chat_id}/attachments", s.auth(s.uploadAttachment))
mux.HandleFunc("/api/v1/attachments/{attachment_id}", s.auth(s.getAttachment))
mux.HandleFunc("/api/v1/attachments/{attachment_id}/content", s.auth(s.downloadAttachment))
mux.HandleFunc("/", s.notFound)
var h http.Handler = mux
h = timeoutMiddleware(s.timeout, s.uploadTimeout, h)
h = recoverMiddleware(s.log, h)
h = loggingMiddleware(s.log, h)
h = requestIDMiddleware(h)
h = instanceIDMiddleware(s.instanceID, h)
return h
}
+64
View File
@@ -0,0 +1,64 @@
package apperr
import (
"encoding/json"
"net/http"
)
type Code string
const (
CodeInvalidRequest Code = "invalid_request"
CodeUnauthorized Code = "unauthorized"
CodeNotFound Code = "not_found"
CodeConflict Code = "conflict"
CodePayloadTooLarge Code = "payload_too_large"
CodeUnsupportedMedia Code = "unsupported_media_type"
CodeRateLimited Code = "rate_limited"
CodeUnavailable Code = "unavailable"
CodeInternal Code = "internal_error"
)
type Body struct {
Code Code `json:"code"`
Message string `json:"message"`
RequestID string `json:"request_id"`
}
type Envelope struct {
Error Body `json:"error"`
}
type AppError struct {
Status int
Code Code
Msg string
}
func (e *AppError) Error() string { return e.Msg }
func New(status int, code Code, msg string) *AppError {
return &AppError{Status: status, Code: code, Msg: msg}
}
func InvalidRequest(msg string) *AppError { return New(http.StatusBadRequest, CodeInvalidRequest, msg) }
func Unauthorized(msg string) *AppError { return New(http.StatusUnauthorized, CodeUnauthorized, msg) }
func NotFound(msg string) *AppError { return New(http.StatusNotFound, CodeNotFound, msg) }
func Conflict(msg string) *AppError { return New(http.StatusConflict, CodeConflict, msg) }
func PayloadTooLarge(msg string) *AppError {
return New(http.StatusRequestEntityTooLarge, CodePayloadTooLarge, msg)
}
func UnsupportedMedia(msg string) *AppError {
return New(http.StatusUnsupportedMediaType, CodeUnsupportedMedia, msg)
}
func Unavailable(msg string) *AppError {
return New(http.StatusServiceUnavailable, CodeUnavailable, msg)
}
func Internal(msg string) *AppError { return New(http.StatusInternalServerError, CodeInternal, msg) }
func Write(w http.ResponseWriter, requestID string, err *AppError) {
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("X-Request-Id", requestID)
w.WriteHeader(err.Status)
_ = json.NewEncoder(w).Encode(Envelope{Error: Body{Code: err.Code, Message: err.Msg, RequestID: requestID}})
}
+100
View File
@@ -0,0 +1,100 @@
package auth
import (
"context"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"errors"
"fmt"
"strings"
"golang.org/x/crypto/argon2"
)
const (
argonMemoryKiB = 19 * 1024
argonTime = 2
argonThreads = 1
argonKeyLen = 32
argonSaltLen = 16
)
var ErrBadHash = errors.New("malformed password hash")
type Hasher struct {
slots chan struct{}
dummy string
}
func NewHasher(concurrency int) *Hasher {
h := &Hasher{slots: make(chan struct{}, concurrency)}
salt := make([]byte, argonSaltLen)
_, _ = rand.Read(salt)
h.dummy = encode(salt, argon2.IDKey([]byte("dummy-password-x"), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen))
return h
}
func (h *Hasher) acquire(ctx context.Context) error {
select {
case h.slots <- struct{}{}:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func (h *Hasher) release() { <-h.slots }
func encode(salt, key []byte) string {
return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", argon2.Version, argonMemoryKiB, argonTime, argonThreads,
base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(key))
}
func (h *Hasher) Hash(ctx context.Context, password string) (string, error) {
if err := h.acquire(ctx); err != nil {
return "", err
}
defer h.release()
salt := make([]byte, argonSaltLen)
if _, err := rand.Read(salt); err != nil {
return "", err
}
return encode(salt, argon2.IDKey([]byte(password), salt, argonTime, argonMemoryKiB, argonThreads, argonKeyLen)), nil
}
func (h *Hasher) Verify(ctx context.Context, password, encoded string) (bool, error) {
missing := encoded == ""
if missing {
encoded = h.dummy
}
parts := strings.Split(encoded, "$")
if len(parts) != 6 || parts[1] != "argon2id" {
return false, ErrBadHash
}
var version int
var mem uint32
var t uint32
var p uint8
if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil || version != argon2.Version {
return false, ErrBadHash
}
if _, err := fmt.Sscanf(parts[3], "m=%d,t=%d,p=%d", &mem, &t, &p); err != nil {
return false, ErrBadHash
}
salt, err := base64.RawStdEncoding.DecodeString(parts[4])
if err != nil {
return false, ErrBadHash
}
want, err := base64.RawStdEncoding.DecodeString(parts[5])
if err != nil {
return false, ErrBadHash
}
if err := h.acquire(ctx); err != nil {
return false, err
}
got := argon2.IDKey([]byte(password), salt, t, mem, p, uint32(len(want)))
h.release()
return subtle.ConstantTimeCompare(got, want) == 1 && !missing, nil
}
+137
View File
@@ -0,0 +1,137 @@
package auth
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"errors"
"log/slog"
"time"
"mtusi-messenger/internal/model"
"mtusi-messenger/internal/store"
)
var (
ErrInvalidSession = errors.New("invalid session")
ErrCacheUnavailable = errors.New("session cache unavailable")
)
type SessionCache interface {
Get(ctx context.Context, key string) (*CachedSession, error)
Set(ctx context.Context, key string, s *CachedSession, ttl time.Duration) error
Del(ctx context.Context, key string) error
}
type CachedSession struct {
User model.User `json:"user"`
ExpiresAt time.Time `json:"exp"`
}
type Sessions struct {
store *store.Store
cache SessionCache
ttl time.Duration
cacheMaxTTL time.Duration
log *slog.Logger
}
func NewSessions(st *store.Store, cache SessionCache, ttl, cacheMaxTTL time.Duration, log *slog.Logger) *Sessions {
return &Sessions{store: st, cache: cache, ttl: ttl, cacheMaxTTL: cacheMaxTTL, log: log}
}
func NewToken() (string, []byte, error) {
raw := make([]byte, 32)
if _, err := rand.Read(raw); err != nil {
return "", nil, err
}
token := base64.RawURLEncoding.EncodeToString(raw)
return token, HashToken(token), nil
}
func HashToken(token string) []byte {
sum := sha256.Sum256([]byte(token))
return sum[:]
}
func cacheKey(hash []byte) string {
return "sess:" + base64.RawURLEncoding.EncodeToString(hash)
}
func (s *Sessions) Create(ctx context.Context, user *model.User) (string, time.Time, error) {
token, hash, err := NewToken()
if err != nil {
return "", time.Time{}, err
}
expires, err := s.store.CreateSession(ctx, hash, user.ID, s.ttl)
if err != nil {
return "", time.Time{}, err
}
s.cacheSet(ctx, hash, user, expires)
return token, expires, nil
}
func (s *Sessions) Resolve(ctx context.Context, token string) (*model.User, error) {
hash := HashToken(token)
if s.cache != nil {
cs, err := s.cache.Get(ctx, cacheKey(hash))
if err == nil && cs != nil {
if time.Now().Before(cs.ExpiresAt) {
u := cs.User
return &u, nil
}
} else if err != nil {
s.log.Debug("session cache get failed, falling back to postgres", "error", err.Error())
}
}
user, expires, err := s.store.SessionUser(ctx, hash)
if errors.Is(err, store.ErrNotFound) {
return nil, ErrInvalidSession
}
if err != nil {
return nil, err
}
s.cacheSet(ctx, hash, user, expires)
return user, nil
}
func (s *Sessions) Revoke(ctx context.Context, token string) error {
hash := HashToken(token)
if s.cache != nil {
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
return ErrCacheUnavailable
}
}
err := s.store.RevokeSession(ctx, hash)
if errors.Is(err, store.ErrNotFound) {
return ErrInvalidSession
}
if err != nil {
return err
}
if s.cache != nil {
if err := s.cache.Del(ctx, cacheKey(hash)); err != nil {
s.log.Warn("session cache delete after revoke failed", "error", err.Error())
}
}
return nil
}
func (s *Sessions) cacheSet(ctx context.Context, hash []byte, user *model.User, expires time.Time) {
if s.cache == nil {
return
}
ttl := time.Until(expires)
if ttl > s.cacheMaxTTL {
ttl = s.cacheMaxTTL
}
if ttl <= 0 {
return
}
if err := s.cache.Set(ctx, cacheKey(hash), &CachedSession{User: *user, ExpiresAt: expires}, ttl); err != nil {
s.log.Debug("session cache set failed", "error", err.Error())
}
}
+80
View File
@@ -0,0 +1,80 @@
package cache
import (
"context"
"encoding/json"
"errors"
"sync/atomic"
"time"
"github.com/redis/go-redis/v9"
"mtusi-messenger/internal/auth"
)
var ErrOpen = errors.New("cache circuit open")
type Valkey struct {
rdb *redis.Client
openUntil atomic.Int64
cooldown time.Duration
}
func NewValkey(url string) (*Valkey, error) {
opt, err := redis.ParseURL(url)
if err != nil {
return nil, err
}
opt.DialTimeout = 500 * time.Millisecond
opt.ReadTimeout = 300 * time.Millisecond
opt.WriteTimeout = 300 * time.Millisecond
opt.PoolSize = 10
opt.MaxRetries = 0
return &Valkey{rdb: redis.NewClient(opt), cooldown: 2 * time.Second}, nil
}
func (v *Valkey) Close() error { return v.rdb.Close() }
func (v *Valkey) Ping(ctx context.Context) error { return v.guard(v.rdb.Ping(ctx).Err()) }
func (v *Valkey) open() bool { return time.Now().UnixNano() < v.openUntil.Load() }
func (v *Valkey) guard(err error) error {
if err != nil && !errors.Is(err, redis.Nil) {
v.openUntil.Store(time.Now().Add(v.cooldown).UnixNano())
}
return err
}
func (v *Valkey) Get(ctx context.Context, key string) (*auth.CachedSession, error) {
if v.open() {
return nil, ErrOpen
}
raw, err := v.rdb.Get(ctx, key).Bytes()
if errors.Is(err, redis.Nil) {
return nil, nil
}
if err := v.guard(err); err != nil {
return nil, err
}
var cs auth.CachedSession
if err := json.Unmarshal(raw, &cs); err != nil {
return nil, nil
}
return &cs, nil
}
func (v *Valkey) Set(ctx context.Context, key string, s *auth.CachedSession, ttl time.Duration) error {
if v.open() {
return ErrOpen
}
raw, err := json.Marshal(s)
if err != nil {
return err
}
return v.guard(v.rdb.Set(ctx, key, raw, ttl).Err())
}
func (v *Valkey) Del(ctx context.Context, key string) error {
return v.guard(v.rdb.Del(ctx, key).Err())
}
+122
View File
@@ -0,0 +1,122 @@
package db
import (
"context"
"embed"
"fmt"
"io/fs"
"log/slog"
"sort"
"strconv"
"strings"
"github.com/jackc/pgx/v5/pgxpool"
)
//go:embed migrations/*.sql
var migrationFS embed.FS
const advisoryLockID = 20260202
type migration struct {
version int
name string
sql string
}
func loadMigrations() ([]migration, error) {
entries, err := fs.ReadDir(migrationFS, "migrations")
if err != nil {
return nil, err
}
var out []migration
for _, e := range entries {
name := e.Name()
prefix, _, ok := strings.Cut(name, "_")
if !ok || !strings.HasSuffix(name, ".sql") {
return nil, fmt.Errorf("bad migration file name %q", name)
}
v, err := strconv.Atoi(prefix)
if err != nil {
return nil, fmt.Errorf("bad migration version in %q", name)
}
body, err := migrationFS.ReadFile("migrations/" + name)
if err != nil {
return nil, err
}
out = append(out, migration{version: v, name: name, sql: string(body)})
}
sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
for i := 1; i < len(out); i++ {
if out[i].version == out[i-1].version {
return nil, fmt.Errorf("duplicate migration version %d", out[i].version)
}
}
return out, nil
}
func Migrate(ctx context.Context, pool *pgxpool.Pool, log *slog.Logger) error {
migs, err := loadMigrations()
if err != nil {
return err
}
conn, err := pool.Acquire(ctx)
if err != nil {
return err
}
defer conn.Release()
if _, err := conn.Exec(ctx, "SELECT pg_advisory_lock($1)", advisoryLockID); err != nil {
return err
}
defer conn.Exec(context.Background(), "SELECT pg_advisory_unlock($1)", advisoryLockID)
if _, err := conn.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
version integer PRIMARY KEY,
name text NOT NULL,
applied_at timestamptz NOT NULL DEFAULT now())`); err != nil {
return err
}
applied := map[int]bool{}
rows, err := conn.Query(ctx, "SELECT version FROM schema_migrations")
if err != nil {
return err
}
for rows.Next() {
var v int
if err := rows.Scan(&v); err != nil {
rows.Close()
return err
}
applied[v] = true
}
rows.Close()
if err := rows.Err(); err != nil {
return err
}
for _, m := range migs {
if applied[m.version] {
continue
}
tx, err := conn.Begin(ctx)
if err != nil {
return err
}
if _, err := tx.Exec(ctx, m.sql); err != nil {
_ = tx.Rollback(ctx)
return fmt.Errorf("migration %s: %w", m.name, err)
}
if _, err := tx.Exec(ctx, "INSERT INTO schema_migrations (version, name) VALUES ($1, $2)", m.version, m.name); err != nil {
_ = tx.Rollback(ctx)
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
log.Info("migration applied", "version", m.version, "name", m.name)
}
log.Info("migrations up to date", "total", len(migs))
return nil
}
+51
View File
@@ -0,0 +1,51 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL,
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000),
CONSTRAINT messages_sender_member_fk FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
expires_at timestamptz NOT NULL,
revoked_at timestamptz,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at)
);
CREATE INDEX sessions_user_idx ON sessions (user_id);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
@@ -0,0 +1,13 @@
CREATE TABLE message_idempotency (
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
idem_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED,
created_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (user_id, chat_id, idem_key),
CONSTRAINT message_idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_hash_len CHECK (octet_length(request_hash) = 32)
);
CREATE INDEX message_idempotency_created_idx ON message_idempotency (created_at);
@@ -0,0 +1,51 @@
-- Lab4: attachment metadata lives here, bytes live in the private S3 bucket.
-- Text may be empty when a message carries at least one attachment (checked by the API).
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
-- status: pending = intent row written before S3 PUT (not visible through the API);
-- ready = object stored and metadata committed.
-- missing_at is set by the reconciler when a ready row has no S3 object.
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL,
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
width integer NOT NULL,
height integer NOT NULL,
object_key text NOT NULL,
status text NOT NULL,
missing_at timestamptz,
created_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
updated_at timestamptz NOT NULL DEFAULT date_trunc('milliseconds', now()),
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
CONSTRAINT attachments_chat_id_key UNIQUE (chat_id, id),
CONSTRAINT attachments_owner_member_fk FOREIGN KEY (chat_id, owner_id)
REFERENCES chat_members (chat_id, user_id) ON DELETE CASCADE,
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_pixels CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready'))
);
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
-- (chat_id, attachment_id) -> attachments (chat_id, id): the database itself refuses
-- to link a file of another chat to a message.
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED,
chat_id uuid NOT NULL,
attachment_id uuid NOT NULL,
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position),
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9),
CONSTRAINT message_attachments_same_chat_fk FOREIGN KEY (chat_id, attachment_id)
REFERENCES attachments (chat_id, id)
);
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
+31
View File
@@ -0,0 +1,31 @@
package db
import (
"context"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
type PoolOptions struct {
MaxConns int32
}
func Open(ctx context.Context, url string, opt PoolOptions) (*pgxpool.Pool, error) {
cfg, err := pgxpool.ParseConfig(url)
if err != nil {
return nil, err
}
cfg.MaxConns = opt.MaxConns
cfg.MinConns = 1
cfg.MaxConnLifetime = 30 * time.Minute
cfg.MaxConnIdleTime = 5 * time.Minute
cfg.HealthCheckPeriod = 15 * time.Second
cfg.ConnConfig.ConnectTimeout = 3 * time.Second
rp := cfg.ConnConfig.RuntimeParams
rp["application_name"] = "mtusi-messenger"
rp["statement_timeout"] = "5000"
rp["lock_timeout"] = "2000"
rp["idle_in_transaction_session_timeout"] = "10000"
return pgxpool.NewWithConfig(ctx, cfg)
}
+79
View File
@@ -0,0 +1,79 @@
package model
import "time"
type Time struct {
time.Time
}
func NewTime(t time.Time) Time { return Time{Time: t} }
func (t Time) MarshalJSON() ([]byte, error) {
return []byte(`"` + t.UTC().Format("2006-01-02T15:04:05.000Z") + `"`), nil
}
type User struct {
ID string `json:"id"`
Username string `json:"username"`
DisplayName string `json:"display_name"`
CreatedAt Time `json:"created_at"`
}
type Chat struct {
ID string `json:"id"`
Title string `json:"title"`
MemberIDs []string `json:"member_ids"`
CreatedAt Time `json:"created_at"`
}
type Message struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
SenderID string `json:"sender_id"`
Text string `json:"text"`
CreatedAt Time `json:"created_at"`
AttachmentIDs []string `json:"attachment_ids"`
}
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type Attachment struct {
ID string `json:"id"`
ChatID string `json:"chat_id"`
OwnerID string `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt Time `json:"created_at"`
UpdatedAt Time `json:"updated_at"`
ObjectKey string `json:"-"`
Missing bool `json:"-"`
}
type Page[T any] struct {
Items []T `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type Health struct {
Status string `json:"status"`
}
type Session struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
ExpiresAt Time `json:"expires_at"`
User User `json:"user"`
}
+151
View File
@@ -0,0 +1,151 @@
package objstore
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
type Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
}
type Store struct {
c *minio.Client
bucket string
}
type Object struct {
Key string
Size int64
LastModified time.Time
}
func New(cfg Config) (*Store, error) {
u, err := url.Parse(cfg.Endpoint)
if err != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
return nil, errors.New("S3_ENDPOINT must be http(s)://host:port")
}
if cfg.Bucket == "" || cfg.AccessKey == "" || cfg.SecretKey == "" {
return nil, errors.New("S3 bucket and credentials are required")
}
// The SDK retries 10 times with backoff by default; a dead S3 must turn into a
// fast 503 instead (the client may retry the upload itself).
minio.MaxRetry = 1
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext,
MaxIdleConnsPerHost: 32,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
}
c, err := minio.New(u.Host, &minio.Options{
Transport: transport,
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: u.Scheme == "https",
Region: cfg.Region,
BucketLookup: minio.BucketLookupPath,
})
if err != nil {
return nil, err
}
return &Store{c: c, bucket: cfg.Bucket}, nil
}
func (s *Store) Bucket() string { return s.bucket }
// Ping checks that the bucket is reachable with the configured credentials.
func (s *Store) Ping(ctx context.Context) error {
ok, err := s.c.BucketExists(ctx, s.bucket)
if err != nil {
return err
}
if !ok {
return fmt.Errorf("bucket %q does not exist", s.bucket)
}
return nil
}
// EnsureBucket creates the bucket if it is absent; safe to repeat.
func (s *Store) EnsureBucket(ctx context.Context, region string) (bool, error) {
ok, err := s.c.BucketExists(ctx, s.bucket)
if err != nil || ok {
return false, err
}
if err := s.c.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: region}); err != nil {
if code := minio.ToErrorResponse(err).Code; code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
return false, nil
}
return false, err
}
return true, nil
}
func (s *Store) Put(ctx context.Context, key string, data []byte, contentType string) error {
_, err := s.c.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
minio.PutObjectOptions{ContentType: contentType, SendContentMd5: true, DisableMultipart: true})
return err
}
// Get returns a stream of the object; Stat is called first so that a missing
// object or an unavailable S3 is reported before any response byte is written.
func (s *Store) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
obj, err := s.c.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, 0, err
}
info, err := obj.Stat()
if err != nil {
obj.Close()
return nil, 0, err
}
return obj, info.Size, nil
}
func (s *Store) Stat(ctx context.Context, key string) (Object, error) {
info, err := s.c.StatObject(ctx, s.bucket, key, minio.StatObjectOptions{})
if err != nil {
return Object{}, err
}
return Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}, nil
}
func (s *Store) Remove(ctx context.Context, key string) error {
return s.c.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{})
}
func (s *Store) List(ctx context.Context, prefix string, fn func(Object) error) error {
for info := range s.c.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) {
if info.Err != nil {
return info.Err
}
if err := fn(Object{Key: info.Key, Size: info.Size, LastModified: info.LastModified}); err != nil {
return err
}
}
return ctx.Err()
}
func IsNotFound(err error) bool {
if err == nil {
return false
}
switch minio.ToErrorResponse(err).Code {
case "NoSuchKey", "NotFound":
return true
}
return false
}
+268
View File
@@ -0,0 +1,268 @@
// Package reconcile compares attachment metadata in PostgreSQL with objects in
// the S3 bucket and repairs what can be repaired safely.
//
// Order matters: S3 is listed first, the database is read second. An upload
// writes its pending row before the PUT, so an object seen in the listing
// already has its row when the database is read. Every destructive decision is
// re-checked against the source of truth right before it is applied.
package reconcile
import (
"context"
"errors"
"log/slog"
"strings"
"time"
"mtusi-messenger/internal/objstore"
"mtusi-messenger/internal/store"
)
const (
Prefix = "attachments/"
lockID = 20260404
)
type Options struct {
Apply bool
Grace time.Duration
Now func() time.Time
}
type Item struct {
Kind string `json:"kind"`
Key string `json:"key,omitempty"`
ID string `json:"attachment_id,omitempty"`
Action string `json:"action"`
}
type Report struct {
Mode string `json:"mode"`
GraceSeconds int64 `json:"grace_seconds"`
ObjectsScanned int `json:"objects_scanned"`
RowsScanned int `json:"rows_scanned"`
OrphanObjects int `json:"orphan_objects"`
FreshSkipped int `json:"fresh_skipped"`
StalePending int `json:"stale_pending"`
MissingObjects int `json:"missing_objects"`
SizeMismatch int `json:"size_mismatch"`
Recovered int `json:"recovered"`
Fixed int `json:"fixed"`
Errors int `json:"errors"`
ListMS int64 `json:"list_ms"`
DBMS int64 `json:"db_ms"`
TotalMS int64 `json:"total_ms"`
Items []Item `json:"items"`
Truncated bool `json:"items_truncated"`
PerObjectUS float64 `json:"per_object_us"`
}
var ErrLocked = errors.New("another reconciler run holds the lock")
type Reconciler struct {
db *store.Store
s3 *objstore.Store
log *slog.Logger
}
func New(db *store.Store, s3 *objstore.Store, log *slog.Logger) *Reconciler {
return &Reconciler{db: db, s3: s3, log: log}
}
func (r *Reconciler) note(rep *Report, it Item) {
if len(rep.Items) < 200 {
rep.Items = append(rep.Items, it)
} else {
rep.Truncated = true
}
}
func (r *Reconciler) Run(ctx context.Context, opt Options) (*Report, error) {
if opt.Now == nil {
opt.Now = time.Now
}
unlock, ok, err := r.db.TryLock(ctx, lockID)
if err != nil {
return nil, err
}
if !ok {
return nil, ErrLocked
}
defer unlock()
rep := &Report{Mode: "dry-run", GraceSeconds: int64(opt.Grace.Seconds()), Items: []Item{}}
if opt.Apply {
rep.Mode = "apply"
}
start := time.Now()
cutoff := opt.Now().Add(-opt.Grace)
objects := make(map[string]objstore.Object)
if err := r.s3.List(ctx, Prefix, func(o objstore.Object) error {
objects[o.Key] = o
return nil
}); err != nil {
return nil, err
}
rep.ObjectsScanned = len(objects)
rep.ListMS = time.Since(start).Milliseconds()
dbStart := time.Now()
known := make(map[string]bool, len(objects))
var refs []store.AttachmentRef
if err := r.db.EachAttachmentRef(ctx, func(a store.AttachmentRef) error {
known[a.ObjectKey] = true
rep.RowsScanned++
obj, present := objects[a.ObjectKey]
switch {
case a.Status == "pending" && a.CreatedAt.Before(cutoff):
refs = append(refs, a)
case a.Status == "ready" && !present && !a.Missing && a.CreatedAt.Before(cutoff):
refs = append(refs, a)
case a.Status == "ready" && present && a.Missing:
refs = append(refs, a)
case a.Status == "ready" && present && obj.Size != a.SizeBytes:
rep.SizeMismatch++
r.note(rep, Item{Kind: "size_mismatch", Key: a.ObjectKey, ID: a.ID, Action: "report only"})
}
return nil
}); err != nil {
return nil, err
}
rep.DBMS = time.Since(dbStart).Milliseconds()
for _, a := range refs {
_, present := objects[a.ObjectKey]
switch {
case a.Status == "pending":
r.stalePending(ctx, rep, a, opt.Apply)
case !present:
r.missing(ctx, rep, a, opt.Apply)
default:
r.recovered(ctx, rep, a, opt.Apply)
}
}
for key, o := range objects {
if known[key] {
continue
}
if !o.LastModified.Before(cutoff) {
rep.FreshSkipped++
r.note(rep, Item{Kind: "orphan_object", Key: key, Action: "skip: younger than grace"})
continue
}
r.orphan(ctx, rep, key, opt.Apply)
}
rep.TotalMS = time.Since(start).Milliseconds()
if rep.ObjectsScanned > 0 {
rep.PerObjectUS = float64(time.Since(start).Microseconds()) / float64(rep.ObjectsScanned)
}
return rep, nil
}
func (r *Reconciler) fail(rep *Report, it Item, err error) {
rep.Errors++
it.Action = "error: " + strings.SplitN(err.Error(), "\n", 2)[0]
r.note(rep, it)
}
// Stale pending row: the upload never reached "ready", so no message can
// reference it. The row is deleted first (conditionally), then the object.
func (r *Reconciler) stalePending(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
rep.StalePending++
it := Item{Kind: "stale_pending", Key: a.ObjectKey, ID: a.ID, Action: "would delete row and object"}
if !apply {
r.note(rep, it)
return
}
deleted, err := r.db.DeletePendingAttachment(ctx, a.ID)
if err != nil {
r.fail(rep, it, err)
return
}
if !deleted {
it.Action = "skip: row changed concurrently"
r.note(rep, it)
return
}
if err := r.s3.Remove(ctx, a.ObjectKey); err != nil {
// The object becomes an orphan and is removed by the next run.
r.fail(rep, it, err)
return
}
rep.Fixed++
it.Action = "deleted row and object"
r.note(rep, it)
}
// Orphan object: no metadata row. The DB is re-checked right before deletion.
func (r *Reconciler) orphan(ctx context.Context, rep *Report, key string, apply bool) {
rep.OrphanObjects++
it := Item{Kind: "orphan_object", Key: key, Action: "would delete object"}
if !apply {
r.note(rep, it)
return
}
known, err := r.db.ObjectKeyKnown(ctx, key)
if err != nil {
r.fail(rep, it, err)
return
}
if known {
it.Action = "skip: row appeared concurrently"
r.note(rep, it)
return
}
if err := r.s3.Remove(ctx, key); err != nil {
r.fail(rep, it, err)
return
}
rep.Fixed++
it.Action = "deleted object"
r.note(rep, it)
}
// Missing object of a ready row: bytes cannot be restored, metadata of a
// possibly referenced attachment is never deleted. The row is marked so that
// operators see it; StatObject confirms the absence first.
func (r *Reconciler) missing(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
_, err := r.s3.Stat(ctx, a.ObjectKey)
if err == nil {
return
}
it := Item{Kind: "missing_object", Key: a.ObjectKey, ID: a.ID, Action: "would mark missing (metadata kept)"}
if !objstore.IsNotFound(err) {
r.fail(rep, it, err)
return
}
rep.MissingObjects++
if !apply {
r.note(rep, it)
return
}
if err := r.db.SetAttachmentMissing(ctx, a.ID, true); err != nil {
r.fail(rep, it, err)
return
}
rep.Fixed++
it.Action = "marked missing (metadata kept)"
r.note(rep, it)
}
func (r *Reconciler) recovered(ctx context.Context, rep *Report, a store.AttachmentRef, apply bool) {
rep.Recovered++
it := Item{Kind: "recovered_object", Key: a.ObjectKey, ID: a.ID, Action: "would clear missing mark"}
if !apply {
r.note(rep, it)
return
}
if err := r.db.SetAttachmentMissing(ctx, a.ID, false); err != nil {
r.fail(rep, it, err)
return
}
rep.Fixed++
it.Action = "cleared missing mark"
r.note(rep, it)
}
+152
View File
@@ -0,0 +1,152 @@
package store
import (
"context"
"errors"
"time"
"github.com/jackc/pgx/v5"
"mtusi-messenger/internal/model"
)
const attachmentColumns = `a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
a.object_key, a.missing_at IS NOT NULL, a.created_at, a.updated_at`
func scanAttachment(row pgx.Row) (*model.Attachment, error) {
a := &model.Attachment{Status: "ready", Variants: []model.Variant{}}
var created, updated time.Time
err := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
&a.ObjectKey, &a.Missing, &created, &updated)
if err != nil {
return nil, err
}
a.CreatedAt, a.UpdatedAt = utc(created), utc(updated)
return a, nil
}
// InsertPendingAttachment records the intent before the S3 PUT. A pending row is
// invisible to the API; if the upload never completes, the reconciler removes
// the row and the object after the grace period.
func (s *Store) InsertPendingAttachment(ctx context.Context, a *model.Attachment, width, height int) error {
var created time.Time
err := s.pool.QueryRow(ctx, `
INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
width, height, object_key, status)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 'pending')
RETURNING created_at`,
a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256,
width, height, a.ObjectKey).Scan(&created)
if pgCode(err) == "23503" {
return ErrNotFound
}
if err != nil {
return err
}
a.CreatedAt = utc(created)
return nil
}
// MarkAttachmentReady succeeds only while the row is still pending: if the
// reconciler already removed a stale intent, the upload must not report 201.
func (s *Store) MarkAttachmentReady(ctx context.Context, id string) (time.Time, error) {
var updated time.Time
err := s.pool.QueryRow(ctx, `
UPDATE attachments SET status = 'ready', updated_at = date_trunc('milliseconds', now())
WHERE id = $1 AND status = 'pending'
RETURNING updated_at`, id).Scan(&updated)
if errors.Is(err, pgx.ErrNoRows) {
return time.Time{}, ErrNotFound
}
if err != nil {
return time.Time{}, err
}
return updated.UTC(), nil
}
// DeletePendingAttachment removes an intent row that never became ready.
func (s *Store) DeletePendingAttachment(ctx context.Context, id string) (bool, error) {
tag, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
if err != nil {
return false, err
}
return tag.RowsAffected() == 1, nil
}
// AttachmentForUser returns a ready attachment only to a member of its chat.
func (s *Store) AttachmentForUser(ctx context.Context, id, userID string) (*model.Attachment, error) {
a, err := scanAttachment(s.pool.QueryRow(ctx, `
SELECT `+attachmentColumns+`
FROM attachments a
WHERE a.id = $1 AND a.status = 'ready'
AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = $2)`,
id, userID))
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return a, err
}
// AttachmentRef is the reconciler's view of one metadata row.
type AttachmentRef struct {
ID string
ObjectKey string
Status string
SizeBytes int64
Missing bool
CreatedAt time.Time
Referenced bool
}
func (s *Store) EachAttachmentRef(ctx context.Context, fn func(AttachmentRef) error) error {
rows, err := s.pool.Query(ctx, `
SELECT a.id, a.object_key, a.status, a.size_bytes, a.missing_at IS NOT NULL, a.created_at,
EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id)
FROM attachments a`)
if err != nil {
return err
}
defer rows.Close()
for rows.Next() {
var r AttachmentRef
if err := rows.Scan(&r.ID, &r.ObjectKey, &r.Status, &r.SizeBytes, &r.Missing, &r.CreatedAt, &r.Referenced); err != nil {
return err
}
if err := fn(r); err != nil {
return err
}
}
return rows.Err()
}
func (s *Store) ObjectKeyKnown(ctx context.Context, key string) (bool, error) {
var ok bool
err := s.pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE object_key = $1)`, key).Scan(&ok)
return ok, err
}
func (s *Store) SetAttachmentMissing(ctx context.Context, id string, missing bool) error {
_, err := s.pool.Exec(ctx, `
UPDATE attachments
SET missing_at = CASE WHEN $2 THEN COALESCE(missing_at, now()) END
WHERE id = $1 AND status = 'ready'`, id, missing)
return err
}
// TryLock takes a session-level advisory lock on a dedicated connection so that
// only one reconciler pass runs at a time. The returned func releases it.
func (s *Store) TryLock(ctx context.Context, id int64) (func(), bool, error) {
conn, err := s.pool.Acquire(ctx)
if err != nil {
return nil, false, err
}
var ok bool
if err := conn.QueryRow(ctx, `SELECT pg_try_advisory_lock($1)`, id).Scan(&ok); err != nil || !ok {
conn.Release()
return nil, false, err
}
return func() {
_, _ = conn.Exec(context.Background(), `SELECT pg_advisory_unlock($1)`, id)
conn.Release()
}, true, nil
}
+33
View File
@@ -0,0 +1,33 @@
package store
import (
"encoding/base64"
"encoding/json"
"time"
)
type Cursor struct {
Scope string `json:"s"`
TS time.Time `json:"t"`
ID string `json:"i"`
}
func EncodeCursor(c Cursor) string {
b, _ := json.Marshal(c)
return base64.RawURLEncoding.EncodeToString(b)
}
func DecodeCursor(s string) (Cursor, error) {
var c Cursor
b, err := base64.RawURLEncoding.DecodeString(s)
if err != nil {
return c, ErrInvalidCursor
}
if err := json.Unmarshal(b, &c); err != nil {
return c, ErrInvalidCursor
}
if c.Scope == "" || c.ID == "" || c.TS.IsZero() {
return c, ErrInvalidCursor
}
return c, nil
}
+385
View File
@@ -0,0 +1,385 @@
package store
import (
"bytes"
"context"
"errors"
"net"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"mtusi-messenger/internal/model"
)
var (
ErrNotFound = errors.New("not found")
ErrUsernameTaken = errors.New("username taken")
ErrUnknownMember = errors.New("unknown member")
ErrInvalidCursor = errors.New("invalid cursor")
)
type Store struct {
pool *pgxpool.Pool
}
func New(pool *pgxpool.Pool) *Store { return &Store{pool: pool} }
func (s *Store) Ping(ctx context.Context) error { return s.pool.Ping(ctx) }
func IsUnavailable(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || pgconn.Timeout(err) {
return true
}
var connErr *pgconn.ConnectError
if errors.As(err, &connErr) {
return true
}
var netErr net.Error
if errors.As(err, &netErr) {
return true
}
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
switch {
case pgErr.Code == "57P01", pgErr.Code == "57P03", pgErr.Code == "57014", pgErr.Code == "55P03",
len(pgErr.Code) == 5 && pgErr.Code[:2] == "08":
return true
}
}
return errors.Is(err, pgx.ErrTxClosed) || errors.Is(err, net.ErrClosed)
}
func pgCode(err error) string {
var pgErr *pgconn.PgError
if errors.As(err, &pgErr) {
return pgErr.Code
}
return ""
}
func utc(t time.Time) model.Time { return model.NewTime(t.UTC()) }
func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (*model.User, error) {
u := &model.User{ID: uuid.NewString(), Username: username, DisplayName: displayName}
var created time.Time
err := s.pool.QueryRow(ctx,
`INSERT INTO users (id, username, display_name, password_hash)
VALUES ($1, $2, $3, $4) RETURNING created_at`,
u.ID, username, displayName, passwordHash).Scan(&created)
if pgCode(err) == "23505" {
return nil, ErrUsernameTaken
}
if err != nil {
return nil, err
}
u.CreatedAt = utc(created)
return u, nil
}
func (s *Store) Credentials(ctx context.Context, username string) (*model.User, string, error) {
u := &model.User{}
var created time.Time
var hash string
err := s.pool.QueryRow(ctx,
`SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1`,
username).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &hash)
if errors.Is(err, pgx.ErrNoRows) {
return nil, "", ErrNotFound
}
if err != nil {
return nil, "", err
}
u.CreatedAt = utc(created)
return u, hash, nil
}
func (s *Store) CreateSession(ctx context.Context, tokenHash []byte, userID string, ttl time.Duration) (time.Time, error) {
var expires time.Time
err := s.pool.QueryRow(ctx,
`INSERT INTO sessions (token_hash, user_id, expires_at)
VALUES ($1, $2, date_trunc('milliseconds', now()) + make_interval(secs => $3))
RETURNING expires_at`,
tokenHash, userID, ttl.Seconds()).Scan(&expires)
if err != nil {
return time.Time{}, err
}
return expires.UTC(), nil
}
func (s *Store) SessionUser(ctx context.Context, tokenHash []byte) (*model.User, time.Time, error) {
u := &model.User{}
var created, expires time.Time
err := s.pool.QueryRow(ctx,
`SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
FROM sessions s JOIN users u ON u.id = s.user_id
WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now()`,
tokenHash).Scan(&u.ID, &u.Username, &u.DisplayName, &created, &expires)
if errors.Is(err, pgx.ErrNoRows) {
return nil, time.Time{}, ErrNotFound
}
if err != nil {
return nil, time.Time{}, err
}
u.CreatedAt = utc(created)
return u, expires.UTC(), nil
}
func (s *Store) RevokeSession(ctx context.Context, tokenHash []byte) error {
tag, err := s.pool.Exec(ctx,
`UPDATE sessions SET revoked_at = now()
WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()`, tokenHash)
if err != nil {
return err
}
if tag.RowsAffected() == 0 {
return ErrNotFound
}
return nil
}
func (s *Store) PurgeSessions(ctx context.Context, olderThan time.Duration) (int64, error) {
tag, err := s.pool.Exec(ctx,
`DELETE FROM sessions
WHERE expires_at < now() - make_interval(secs => $1)
OR revoked_at < now() - make_interval(secs => $1)`, olderThan.Seconds())
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
func (s *Store) CreateChat(ctx context.Context, creatorID, title string, memberIDs []string) (*model.Chat, error) {
c := &model.Chat{ID: uuid.NewString(), Title: title, MemberIDs: append([]string(nil), memberIDs...)}
err := pgx.BeginFunc(ctx, s.pool, func(tx pgx.Tx) error {
var created time.Time
if err := tx.QueryRow(ctx,
`INSERT INTO chats (id, title, created_by) VALUES ($1, $2, $3) RETURNING created_at`,
c.ID, title, creatorID).Scan(&created); err != nil {
return err
}
c.CreatedAt = utc(created)
_, err := tx.Exec(ctx,
`INSERT INTO chat_members (chat_id, user_id, position)
SELECT $1, m.user_id, m.ord - 1
FROM unnest($2::uuid[]) WITH ORDINALITY AS m(user_id, ord)`,
c.ID, memberIDs)
return err
})
if pgCode(err) == "23503" {
return nil, ErrUnknownMember
}
if err != nil {
return nil, err
}
return c, nil
}
func (s *Store) IsMember(ctx context.Context, chatID, userID string) (bool, error) {
var ok bool
err := s.pool.QueryRow(ctx,
`SELECT EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`,
chatID, userID).Scan(&ok)
return ok, err
}
func (s *Store) ChatForUser(ctx context.Context, chatID, userID string) (*model.Chat, error) {
c := &model.Chat{}
var created time.Time
err := s.pool.QueryRow(ctx,
`SELECT c.id, c.title, c.created_at,
array_agg(m.user_id::text ORDER BY m.position)
FROM chats c JOIN chat_members m ON m.chat_id = c.id
WHERE c.id = $1
AND EXISTS (SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = $2)
GROUP BY c.id`,
chatID, userID).Scan(&c.ID, &c.Title, &created, &c.MemberIDs)
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
c.CreatedAt = utc(created)
return c, nil
}
func (s *Store) ListChats(ctx context.Context, userID string, limit int, cur *Cursor) ([]*model.Chat, bool, error) {
var ts *time.Time
var cid *string
if cur != nil {
ts, cid = &cur.TS, &cur.ID
}
rows, err := s.pool.Query(ctx,
`SELECT c.id, c.title, c.created_at,
(SELECT array_agg(m.user_id::text ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id)
FROM chat_members me JOIN chats c ON c.id = me.chat_id
WHERE me.user_id = $1
AND ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3::uuid))
ORDER BY c.created_at, c.id
LIMIT $4`,
userID, ts, cid, limit+1)
if err != nil {
return nil, false, err
}
defer rows.Close()
out := make([]*model.Chat, 0, limit+1)
for rows.Next() {
c := &model.Chat{}
var created time.Time
if err := rows.Scan(&c.ID, &c.Title, &created, &c.MemberIDs); err != nil {
return nil, false, err
}
c.CreatedAt = utc(created)
out = append(out, c)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
more := len(out) > limit
if more {
out = out[:limit]
}
return out, more, nil
}
var (
ErrIdempotencyConflict = errors.New("idempotency key reused with different body")
ErrAttachmentNotFound = errors.New("attachment not found in chat")
)
// AddMessage inserts the message, its idempotency record and attachment links in
// one statement. Attachments must be ready and belong to the same chat; the
// composite FK on message_attachments enforces the chat boundary in the DB too.
func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string, attachmentIDs []string, key string, reqHash []byte, ttl time.Duration) (*model.Message, bool, error) {
for attempt := 0; attempt < 2; attempt++ {
newID := uuid.NewString()
var member, attachmentsOK bool
var created *time.Time
err := s.pool.QueryRow(ctx, `
WITH member AS (
SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $1
), att AS (
SELECT count(*) = cardinality($8::uuid[]) AS ok
FROM attachments
WHERE chat_id = $2 AND id = ANY($8::uuid[]) AND status = 'ready'
), k AS (
INSERT INTO message_idempotency (user_id, chat_id, idem_key, request_hash, message_id)
SELECT $1, $2, $3, $4, $5 FROM member, att WHERE att.ok
ON CONFLICT (user_id, chat_id, idem_key) DO UPDATE
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = now()
WHERE message_idempotency.created_at < now() - make_interval(secs => $7)
RETURNING message_id
), m AS (
INSERT INTO messages (id, chat_id, sender_id, text)
SELECT $5, $2, $1, $6 FROM k
RETURNING created_at
), ma AS (
INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
SELECT $5, $2, a.id, a.ord - 1
FROM unnest($8::uuid[]) WITH ORDINALITY AS a(id, ord)
WHERE EXISTS (SELECT 1 FROM m)
)
SELECT EXISTS (SELECT 1 FROM member), (SELECT ok FROM att), (SELECT created_at FROM m)`,
senderID, chatID, key, reqHash, newID, text, ttl.Seconds(), attachmentIDs).Scan(&member, &attachmentsOK, &created)
if pgCode(err) == "23503" {
return nil, false, ErrAttachmentNotFound
}
if err != nil {
return nil, false, err
}
if !member {
return nil, false, ErrNotFound
}
if !attachmentsOK {
return nil, false, ErrAttachmentNotFound
}
if created != nil {
return &model.Message{ID: newID, ChatID: chatID, SenderID: senderID, Text: text,
CreatedAt: utc(*created), AttachmentIDs: attachmentIDs}, false, nil
}
var stored []byte
var at time.Time
m := &model.Message{ChatID: chatID}
err = s.pool.QueryRow(ctx, `
SELECT i.request_hash, m.id, m.sender_id, m.text, m.created_at,
`+attachmentIDsSQL+`
FROM message_idempotency i JOIN messages m ON m.id = i.message_id
WHERE i.user_id = $1 AND i.chat_id = $2 AND i.idem_key = $3
AND i.created_at >= now() - make_interval(secs => $4)`,
senderID, chatID, key, ttl.Seconds()).Scan(&stored, &m.ID, &m.SenderID, &m.Text, &at, &m.AttachmentIDs)
if errors.Is(err, pgx.ErrNoRows) {
continue
}
if err != nil {
return nil, false, err
}
if !bytes.Equal(stored, reqHash) {
return nil, false, ErrIdempotencyConflict
}
m.CreatedAt = utc(at)
return m, true, nil
}
return nil, false, errors.New("idempotency record kept changing")
}
const attachmentIDsSQL = `COALESCE((SELECT array_agg(ma.attachment_id::text ORDER BY ma.position)
FROM message_attachments ma WHERE ma.message_id = m.id), '{}')`
func (s *Store) PurgeIdempotency(ctx context.Context, ttl time.Duration) (int64, error) {
tag, err := s.pool.Exec(ctx,
`DELETE FROM message_idempotency WHERE created_at < now() - make_interval(secs => $1)`, ttl.Seconds())
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, cur *Cursor) ([]*model.Message, bool, error) {
var (
rows pgx.Rows
err error
)
if cur == nil {
rows, err = s.pool.Query(ctx,
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
FROM messages m
WHERE m.chat_id = $1
ORDER BY m.created_at, m.id LIMIT $2`, chatID, limit+1)
} else {
rows, err = s.pool.Query(ctx,
`SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, `+attachmentIDsSQL+`
FROM messages m
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3::uuid)
ORDER BY m.created_at, m.id LIMIT $4`, chatID, cur.TS, cur.ID, limit+1)
}
if err != nil {
return nil, false, err
}
defer rows.Close()
out := make([]*model.Message, 0, limit+1)
for rows.Next() {
m := &model.Message{}
var created time.Time
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &created, &m.AttachmentIDs); err != nil {
return nil, false, err
}
m.CreatedAt = utc(created)
out = append(out, m)
}
if err := rows.Err(); err != nil {
return nil, false, err
}
more := len(out) > limit
if more {
out = out[:limit]
}
return out, more, nil
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

+34
View File
@@ -0,0 +1,34 @@
"""Synthetic PNG builders for negative/limit tests (stdlib only, nothing stored in Git)."""
import os
import struct
import zlib
def chunk(kind, data):
body = kind + data
return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff)
def png(width, height, raw=None, level=6, color_type=2):
"""RGB (color_type=2) PNG; raw is the filtered scanline stream (default: random noise)."""
if raw is None:
row = 1 + 3 * width
raw = b''.join(b'\x00' + os.urandom(row - 1) for _ in range(height))
ihdr = struct.pack('>IIBBBBB', width, height, 8, color_type, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) +
chunk(b'IDAT', zlib.compress(raw, level)) + chunk(b'IEND', b''))
def near_limit_png(limit=10 * 1024 * 1024):
"""Valid noise PNG a little below the byte limit (stored deflate blocks)."""
width = 1000
height = (limit - 64 * 1024) // (1 + 3 * width)
data = png(width, height, level=0)
assert len(data) <= limit, len(data)
return data
def bomb_png(width=20000, height=20000):
"""Header declares 400M pixels, the compressed payload is tiny."""
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) +
chunk(b'IDAT', zlib.compress(b'\x00' * 4096, 9)) + chunk(b'IEND', b''))
+175
View File
@@ -0,0 +1,175 @@
#!/usr/bin/env python3
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import threading
import uuid
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, Failure, check
sys.path.insert(0, str(Path(__file__).resolve().parent))
import test_attachments
RESULTS = []
def case(name):
def wrap(fn):
def run(*a):
try:
detail = fn(*a)
RESULTS.append({'test': name, 'result': 'PASS', 'detail': detail})
print('PASS', name, '-', detail or '')
except Failure as exc:
RESULTS.append({'test': name, 'result': 'FAIL', 'detail': str(exc)})
print('FAIL', name, '-', exc)
return run
return wrap
def msg_path(chat):
return '/api/v1/chats/' + chat['id'] + '/messages'
def history(c, actor, chat):
out, cursor = [], None
while True:
q = '?limit=100' + ('&cursor=' + cursor if cursor else '')
page = c.call('GET', msg_path(chat) + q, actor=actor, schema='MessagePage')['json']
out += page['items']
cursor = page['next_cursor']
if not cursor:
return out
@case('one session served by >=2 replicas without re-login (no stickiness)')
def t_no_sticky(c, alice, bob, chat):
seen = {}
for _ in range(30):
iid = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
seen[iid] = seen.get(iid, 0) + 1
check(len(seen) >= 2, 'only one replica observed: %s' % seen)
return 'distribution %s' % sorted(seen.values())
@case('X-Instance-Id is present on errors and health too')
def t_instance_on_errors(c, alice, bob, chat):
r = c.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice)
check(bool(r['headers'].get('X-Instance-Id')), 'missing on 400')
r = c.call('GET', '/api/v1/users/me', 401)
check(bool(r['headers'].get('X-Instance-Id')), 'missing on 401')
r = c.call('GET', '/health/ready', schema='Health')
check(bool(r['headers'].get('X-Instance-Id')), 'missing on health')
@case('Idempotency-Key missing or malformed -> 400, nothing stored')
def t_key_validation(c, alice, bob, chat):
before = len(history(c, alice, chat))
c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'без ключа'})
for bad in ('', 'a b', 'clé', 'x' * 129, 'a/b'):
c.call('POST', msg_path(chat), 400, actor=alice, body={'text': 'плохой ключ'},
headers={'Idempotency-Key': bad})
c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'граница ключа'},
headers={'Idempotency-Key': 'A-z0_9.:' + 'k' * 120})
check(len(history(c, alice, chat)) == before + 1, 'rejected requests created messages')
@case('concurrent replay of one key through different replicas -> exactly one message')
def t_concurrent_same_key(c, alice, bob, chat):
key = 'conc-' + uuid.uuid4().hex
text = 'Конкурентный повтор ' + key
n = 24
barrier = threading.Barrier(n)
raw = json.dumps({'text': text}, ensure_ascii=False).encode()
def send(_):
local = Client(c.base, timeout=10)
barrier.wait()
r = local.call('POST', msg_path(chat), 201, actor=alice, raw=raw,
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
return r['json'], r['headers']['X-Instance-Id']
with ThreadPoolExecutor(max_workers=n) as pool:
results = list(pool.map(send, range(n)))
bodies = {json.dumps(m, sort_keys=True) for m, _ in results}
instances = {i for _, i in results}
check(len(bodies) == 1, 'replays returned different messages: %d' % len(bodies))
check(len(instances) >= 2, 'all requests hit one replica; test does not prove cross-replica')
same = [m for m in history(c, bob, chat) if m['text'] == text]
check(len(same) == 1, 'expected exactly 1 stored message, found %d' % len(same))
return '%d parallel requests via %d replicas -> 1 message' % (n, len(instances))
@case('replay is semantic: JSON key order/whitespace does not matter; other body -> 409')
def t_semantic_replay(c, alice, bob, chat):
key = uuid.uuid4().hex
first = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{"text":"abc"}',
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
again = c.call('POST', msg_path(chat), 201, actor=alice, raw=b'{ "text" : "abc" }\n',
headers={'Idempotency-Key': key, 'Content-Type': 'application/json'}, schema='Message')
check(first['json'] == again['json'], 'replay returned another message')
c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'abd'}, headers={'Idempotency-Key': key})
third = c.call('POST', msg_path(chat), 201, actor=alice, body={'text': 'abc'},
headers={'Idempotency-Key': key}, schema='Message')
check(third['json'] == first['json'], 'stored result changed after 409')
@case('key scope = user + chat: same key is independent for other user / other chat')
def t_key_scope(c, alice, bob, chat):
key = 'shared-' + uuid.uuid4().hex
a = c.message(alice, chat, 'scope', key=key)['json']
b = c.message(bob, chat, 'scope', key=key)['json']
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Второй', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
a2 = c.message(alice, chat2, 'scope', key=key)['json']
check(len({a['id'], b['id'], a2['id']}) == 3, 'scopes collided')
check(b['sender_id'] == bob['id'], 'sender mixed up')
@case('replay re-checks the session and chat access')
def t_replay_rechecks_access(c, alice, bob, chat, eve):
key = uuid.uuid4().hex
c.message(alice, chat, 'access', key=key)
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'access'}, headers={'Idempotency-Key': key})
c.call('POST', msg_path(chat), 401, body={'text': 'access'}, headers={'Idempotency-Key': key})
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': ' '}, headers={'Idempotency-Key': key})
c.call('POST', msg_path(chat), 404, actor=eve, body={'text': 'x'})
@case('logout is visible on every replica immediately (shared sessions)')
def t_logout_shared(c, alice, bob, chat):
tmp = c.user('tmp')
c.call('GET', '/api/v1/users/me', actor=tmp, schema='User')
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=tmp)
seen = set()
for _ in range(10):
seen.add(c.call('GET', '/api/v1/users/me', 401, actor=tmp)['headers']['X-Instance-Id'])
check(len(seen) >= 2, 'revocation checked on one replica only')
return 'revoked token rejected by %d replicas' % len(seen)
def main():
import argparse
p = argparse.ArgumentParser()
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--json', default=None)
a = p.parse_args()
c = Client(a.base_url, timeout=60)
alice, bob, eve, chat = c.bootstrap()
for t in (t_no_sticky, t_instance_on_errors, t_key_validation, t_concurrent_same_key,
t_semantic_replay, t_key_scope, t_logout_shared):
t(c, alice, bob, chat)
t_replay_rechecks_access(c, alice, bob, chat, eve)
for needs_eve, t in test_attachments.register(case, history, msg_path):
t(c, alice, bob, chat, *([eve] if needs_eve else []))
failed = [r for r in RESULTS if r['result'] != 'PASS']
print('%d/%d passed' % (len(RESULTS) - len(failed), len(RESULTS)))
if a.json:
Path(a.json).write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
sys.exit(1 if failed else 0)
if __name__ == '__main__':
main()
+181
View File
@@ -0,0 +1,181 @@
"""Lab4 attachment cases; registered by run_tests.py (same case() harness)."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import os
from pathlib import Path
import threading
import uuid
from urllib.parse import quote
from client import Client, check
import imagegen
FIXTURES = Path(__file__).resolve().parent / 'fixtures'
PNG = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
JPG = (FIXTURES / 'sample.jpg').read_bytes()
def register(case, history, msg_path):
"""Return the list of (needs_eve, fn) cases bound to the runner's helpers."""
def upload(c, actor, chat, data, ctype='image/png', name='sample.png', expected=201):
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + quote(name, safe=''),
expected, actor=actor, raw=data, headers={'Content-Type': ctype},
schema='Attachment' if expected == 201 else None)
return r['json']
@case('PNG and JPEG upload/download byte-exact with true Content-Type')
def t_upload_formats(c, alice, bob, chat):
for data, ctype, name in ((PNG, 'image/png', 'a.png'), (JPG, 'image/jpeg', 'b.jpg')):
att = upload(c, alice, chat, data, ctype, name)
check(att['content_type'] == ctype and att['size_bytes'] == len(data), 'metadata mismatch')
check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'sha256 mismatch')
check(att['status'] == 'ready' and att['variants'] == [] and att['error_code'] is None, 'lab4 state')
meta = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
check(meta == att, 'GET metadata differs from POST')
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)
check(got['bytes'] == data and got['headers'].get_content_type() == ctype, 'download differs')
check(got['headers'].get('X-Content-Type-Options') == 'nosniff', 'nosniff missing')
return 'png %d B, jpeg %d B' % (len(PNG), len(JPG))
@case('size limit: valid ~10 MiB image accepted, 10 MiB + 1 byte -> 413')
def t_size_limit(c, alice, bob, chat):
near = imagegen.near_limit_png()
att = upload(c, alice, chat, near, name='near-limit.png')
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=alice)
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'] == hashlib.sha256(near).hexdigest(),
'big file changed')
over = PNG + b'\0' * (10 * 1024 * 1024 + 1 - len(PNG))
upload(c, alice, chat, over, name='over.png', expected=413)
return 'accepted %d B, rejected %d B' % (len(near), len(over))
@case('decompression bomb (400M px header, tiny body) -> 413 before decoding')
def t_bomb(c, alice, bob, chat):
data = imagegen.bomb_png()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=bomb.png', 413,
actor=alice, raw=data, headers={'Content-Type': 'image/png'})
check(r['elapsed'] < 2, 'bomb took %.2fs: was it decoded?' % r['elapsed'])
return '%d B rejected in %.0f ms' % (len(data), r['elapsed'] * 1000)
@case('corrupted images and MIME spoofing -> 415, never 500')
def t_corrupt_and_spoof(c, alice, bob, chat):
cases = {
'truncated png': (PNG[:len(PNG) // 2], 'image/png'),
'garbage after png header': (PNG[:40] + os.urandom(2000), 'image/png'),
'truncated jpeg': (JPG[:len(JPG) // 2], 'image/jpeg'),
'png declared as jpeg': (PNG, 'image/jpeg'),
'jpeg declared as png': (JPG, 'image/png'),
'gif bytes as png': (b'GIF89a' + os.urandom(100), 'image/png'),
'html as png': (b'<html><script>alert(1)</script></html>', 'image/png'),
'text/plain content type': (PNG, 'text/plain'),
'multipart content type': (PNG, 'multipart/form-data; boundary=x'),
}
for name, (data, ctype) in cases.items():
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=x.png', 415,
actor=alice, raw=data, headers={'Content-Type': ctype})
check(r['json']['error']['code'] == 'unsupported_media_type', name)
upload(c, alice, chat, b'', name='empty.png', expected=400)
return '%d spoof/corrupt variants -> 415' % len(cases)
@case('dangerous filenames -> 400; Unicode display name kept as-is')
def t_filenames(c, alice, bob, chat):
bad = ['../etc/passwd', 'a/b.png', 'a\\b.png', 'nul\x00.png', 'bell\x07.png', 'line\nbreak.png',
'', ' ', '.', '..', 'x' * 256, 'evil‮gnp.exe']
for name in bad:
upload(c, alice, chat, PNG, name=name, expected=400)
base = '/api/v1/chats/' + chat['id'] + '/attachments'
c.call('POST', base, 400, actor=alice, raw=PNG, headers={'Content-Type': 'image/png'})
c.call('POST', base + '?filename=a.png&filename=b.png', 400, actor=alice, raw=PNG,
headers={'Content-Type': 'image/png'})
nice = 'Фото кота 🐱 (1).png'
att = upload(c, alice, chat, PNG, name=nice)
check(att['filename'] == nice, 'filename changed')
long_ok = upload(c, alice, chat, PNG, name='я' * 255)
check(len(long_ok['filename']) == 255, '255 code points must pass')
return '%d bad names rejected' % len(bad)
@case('ACL: outsider gets 404 on upload, metadata, content; no token -> 401')
def t_attachment_acl(c, alice, bob, chat, eve):
att = upload(c, alice, chat, PNG)
upload(c, eve, chat, PNG, expected=404)
for suffix in ('', '/content', '/content?variant=original'):
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401)
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()), 404, actor=alice)
c.call('GET', '/api/v1/attachments/not-a-uuid', 400, actor=alice)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=alice)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=huge', 400, actor=alice)
@case('message attachment_ids: other chat / foreign / unknown -> 404, nothing stored')
def t_message_links(c, alice, bob, chat, eve):
# alice is a member of both chats: the file still must not cross the chat boundary
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
other = upload(c, alice, chat2, PNG)
eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve,
body={'title': 'Чат Евы', 'member_ids': [eve['id']]}, schema='Chat')['json']
foreign = upload(c, eve, eve_chat, PNG)
own = upload(c, alice, chat, PNG)
before = len(history(c, alice, chat))
for ids in ([other['id']], [foreign['id']], [str(uuid.uuid4())], [own['id'], other['id']]):
r = c.call('POST', msg_path(chat), 404, actor=alice, body={'text': 'x', 'attachment_ids': ids},
headers={'Idempotency-Key': uuid.uuid4().hex})
check(r['json']['error']['code'] == 'not_found', 'expected not_found')
check(len(history(c, alice, chat)) == before, 'rejected message was stored')
m = c.message(bob, chat, '', attachments=[own['id']])['json']
check(m['attachment_ids'] == [own['id']] and m['text'] == '', 'bob could not reference chat file')
listed = [x for x in history(c, alice, chat) if x['id'] == m['id']]
check(listed == [m], 'history lost attachment_ids')
@case('attachment_ids validation: duplicates, >10, bad UUID, empty text without files -> 400')
def t_message_validation(c, alice, bob, chat):
att = upload(c, alice, chat, PNG)
for body in ({'text': 'x', 'attachment_ids': [att['id'], att['id']]},
{'text': 'x', 'attachment_ids': [str(uuid.uuid4()) for _ in range(11)]},
{'text': 'x', 'attachment_ids': ['nope']},
{'text': 'x', 'attachment_ids': 'nope'},
{'text': '', 'attachment_ids': []},
{'text': ' '}):
c.call('POST', msg_path(chat), 400, actor=alice, body=body,
headers={'Idempotency-Key': uuid.uuid4().hex})
ten = [upload(c, alice, chat, PNG)['id'] for _ in range(10)]
m = c.message(alice, chat, 'десять файлов', attachments=ten)['json']
check(m['attachment_ids'] == ten, 'order of attachment_ids changed')
@case('idempotency: missing attachment_ids == []; other attachment list -> 409')
def t_attachment_idempotency(c, alice, bob, chat):
key = uuid.uuid4().hex
first = c.message(alice, chat, 'idem', key=key)['json']
again = c.message(alice, chat, 'idem', key=key, attachments=[])['json']
check(first == again and first['attachment_ids'] == [], 'missing vs [] not equal')
att = upload(c, alice, chat, PNG)
c.call('POST', msg_path(chat), 409, actor=alice, body={'text': 'idem', 'attachment_ids': [att['id']]},
headers={'Idempotency-Key': key})
key2 = uuid.uuid4().hex
a = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json']
b = c.message(alice, chat, '', key=key2, attachments=[att['id']])['json']
check(a == b, 'replay with attachments differs')
@case('10 concurrent uploads through both replicas: all 201, distinct ids, same bytes back')
def t_concurrent_uploads(c, alice, bob, chat):
n = 10
barrier = threading.Barrier(n)
def one(i):
local = Client(c.base, timeout=60)
barrier.wait()
r = local.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=p%d.png' % i, 201,
actor=alice, raw=PNG, headers={'Content-Type': 'image/png'}, schema='Attachment')
return r['json'], r['headers']['X-Instance-Id']
with ThreadPoolExecutor(max_workers=n) as pool:
res = list(pool.map(one, range(n)))
check(len({a['id'] for a, _ in res}) == n, 'ids collided')
for a, _ in res:
got = c.call('GET', '/api/v1/attachments/' + a['id'] + '/content', actor=bob)
check(got['bytes'] == PNG, 'content mixed up')
return '%d uploads via %d replicas' % (n, len({i for _, i in res}))
return [(False, t_upload_formats), (False, t_size_limit), (False, t_bomb), (False, t_corrupt_and_spoof),
(False, t_filenames), (False, t_message_validation), (False, t_attachment_idempotency),
(False, t_concurrent_uploads), (True, t_attachment_acl), (True, t_message_links)]