forked from mtusi2026/mtusi2026lab4
Compare commits
3
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
12805025ba | ||
|
|
02a70b30eb | ||
|
|
53ea5b2faf |
+4
-17
@@ -1,20 +1,7 @@
|
|||||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
HOST_PORT=8084
|
||||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
|
||||||
APP_HOST=0.0.0.0
|
|
||||||
APP_PORT=8080
|
|
||||||
APP_ENV=lab
|
|
||||||
SESSION_TTL_SECONDS=3600
|
SESSION_TTL_SECONDS=3600
|
||||||
# Choose credentials locally. File paths contain no secret values.
|
|
||||||
DATABASE_URL_FILE=/run/secrets/database_url
|
|
||||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
|
||||||
SESSION_STORE=postgres
|
|
||||||
CACHE_URL_FILE=/run/secrets/cache_url
|
|
||||||
API_REPLICAS=2
|
API_REPLICAS=2
|
||||||
IDEMPOTENCY_TTL_SECONDS=86400
|
IDEMPOTENCY_TTL_SECONDS=86400
|
||||||
S3_ENDPOINT=http://s3:9000
|
# scripts/setup.sh generates DB/S3 secrets under secrets/.
|
||||||
S3_REGION=us-east-1
|
# S3 endpoint, bucket and secret-file paths are wired in compose.yaml.
|
||||||
S3_BUCKET=messenger-attachments
|
# Upload limits are fixed: 10485760 bytes / 40000000 pixels.
|
||||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
|
||||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
|
||||||
MAX_UPLOAD_BYTES=10485760
|
|
||||||
MAX_IMAGE_PIXELS=40000000
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
name: Application
|
||||||
|
on: [push, pull_request]
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
jobs:
|
||||||
|
application:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 15
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.13'
|
||||||
|
- run: python -m pip install Pillow==13.0.0 -r requirements-dev.txt
|
||||||
|
- run: scripts/setup.sh && docker compose up -d --build --wait
|
||||||
|
- run: make check validate test BASE_URL=http://localhost:8084
|
||||||
|
- run: python student-tests/check.py --base-url http://localhost:8084
|
||||||
|
- run: python student-tests/idempotency.py --base-url http://localhost:8084
|
||||||
|
- run: python student-tests/attachments.py --base-url http://localhost:8084
|
||||||
|
- run: docker compose exec -T api python - < student-tests/storage.py
|
||||||
|
- run: make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
- run: make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
- if: always()
|
||||||
|
run: docker compose down
|
||||||
+10
@@ -0,0 +1,10 @@
|
|||||||
|
FROM python:3.13.7-slim-bookworm
|
||||||
|
WORKDIR /app
|
||||||
|
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
|
||||||
|
COPY requirements.txt .
|
||||||
|
RUN pip install --no-cache-dir -r requirements.txt
|
||||||
|
COPY src/ src/
|
||||||
|
COPY migrations/ migrations/
|
||||||
|
USER 10001:10001
|
||||||
|
EXPOSE 8080
|
||||||
|
CMD ["python", "src/server.py"]
|
||||||
@@ -1,37 +1,99 @@
|
|||||||
# Отчёт — лабораторная 4
|
# Отчёт — лабораторная 4
|
||||||
|
|
||||||
- Автор, группа: TODO
|
- Автор: **Владислав Гавриленко, БПИ2303**.
|
||||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
- Целевая оценка: **3**, исследовательские треки 4/5 не заявлены.
|
||||||
- Commit/PR: TODO после публикации
|
- Ветка: `solution/bpi2303-gavrilenko`, форк: https://gitea.privcloud.tech/sxdees/mtusi2026lab4.
|
||||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
- MR: [ЛР 4 — БПИ2303 — Гавриленко — на 3, #124](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/124).
|
||||||
|
- Основа: собственная [лабораторная 3](https://gitea.privcloud.tech/sxdees/mtusi2026lab3/src/commit/51dae8c44a93c4cd5488be89ac84146d7e9cb61c), commit `51dae8c44a93c4cd5488be89ac84146d7e9cb61c`.
|
||||||
|
- Источники: контракт и публичные проверки этой лабораторной; [SeaweedFS S3](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API), [Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), [boto3 S3](https://docs.aws.amazon.com/boto3/latest/reference/services/s3.html). Код и отчёт подготовлены с помощью Codex.
|
||||||
|
|
||||||
## Запуск из чистого clone
|
## Запуск из чистого clone
|
||||||
|
|
||||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
Нужны Docker Engine с Compose v2, Python 3.9+ для публичных проверок, свободный порт 8084 и несколько GiB свободного места **в Linux-диске Docker**. Проверено на macOS arm64 / OrbStack, Docker Engine 29.8.2. Образы: Python 3.13.7 slim-bookworm, PostgreSQL 17.6-bookworm, HAProxy 3.2.6-alpine, SeaweedFS 4.48; runtime: psycopg 3.2.10, boto3 1.44.2, Pillow 13.0.0.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git clone -b solution/bpi2303-gavrilenko https://gitea.privcloud.tech/sxdees/mtusi2026lab4.git
|
||||||
|
cd mtusi2026lab4
|
||||||
|
scripts/setup.sh
|
||||||
|
docker compose config --quiet
|
||||||
|
docker compose up -d --build --wait
|
||||||
|
curl --fail http://localhost:8084/health/ready
|
||||||
|
make check test BASE_URL=http://localhost:8084
|
||||||
|
python3 student-tests/check.py --base-url http://localhost:8084
|
||||||
|
python3 student-tests/idempotency.py --base-url http://localhost:8084
|
||||||
|
python3 -m venv .venv
|
||||||
|
.venv/bin/pip install Pillow==13.0.0 -r requirements-dev.txt
|
||||||
|
make validate PYTHON=.venv/bin/python
|
||||||
|
.venv/bin/python student-tests/attachments.py --base-url http://localhost:8084
|
||||||
|
docker compose exec -T api python - < student-tests/storage.py
|
||||||
|
docker compose exec -T api python - < student-tests/db_check.py
|
||||||
|
make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
|
||||||
|
make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
|
||||||
|
docker compose up -d --wait
|
||||||
|
python3 student-tests/attachment_failover.py --base-url http://localhost:8084
|
||||||
|
python3 student-tests/s3_outage.py --base-url http://localhost:8084
|
||||||
|
```
|
||||||
|
|
||||||
|
`setup.sh` создаёт случайные DB/S3 credentials в игнорируемом каталоге `secrets/` с правами 0700, локальный `.env`; существующие файлы сохраняются. Файлы, читаемые непривилегированным API UID 10001 через Docker secrets, имеют права 0644 внутри защищённого каталога. Не копировать `.env`/`secrets/` в Git. Для другого порта изменить `HOST_PORT` и `BASE_URL`.
|
||||||
|
|
||||||
|
Миграции выполняет одноразовый сервис `migrate`: PostgreSQL advisory lock, журнал `schema_migrations`, все изменения атомарны. Одноразовый `bucket` ждёт готовности S3, создаёт bucket при отсутствии и убирает bucket policy с публичными разрешениями; повторный запуск безопасен. `NoSuchBucketPolicy` означает уже приватное состояние. Compose запускает две API-реплики после завершения обоих заданий.
|
||||||
|
|
||||||
|
Безопасная остановка с сохранением данных: `docker compose down`. Повторный запуск: `docker compose up -d --wait`. **Намеренное удаление всех учебных данных этого проекта:** `docker compose down -v`, затем `rm -rf data/s3` (для persistence не использовать). Другие Compose-проекты эти команды не затрагивают.
|
||||||
|
|
||||||
## Архитектура и решения
|
## Архитектура и решения
|
||||||
|
|
||||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
`клиент → 127.0.0.1:8084 HAProxy → API × 2 → PostgreSQL + SeaweedFS S3`.
|
||||||
|
|
||||||
|
Из lab3 перенесены исходники сервера/БД, первые две миграции, Dockerfile, HAProxy DNS service discovery, настройки двух реплик, скрипты и собственные проверки. README, COURSE, contracts, lab.json и публичные tests сохранены из lab4. PostgreSQL хранит пользователей, scrypt-хеши паролей, SHA-256 bearer-токенов с expiry, чаты/членство, сообщения, результаты идемпотентных POST и метаданные вложений. Бинарные оригиналы находятся только в приватном S3 bucket `messenger-attachments`.
|
||||||
|
|
||||||
|
Новая миграция `003_attachments.sql` создаёт Attachment и разрешает сообщение с пустым текстом при непустом `attachment_ids`. Вложение принадлежит чату и загрузившему участнику; серверный ключ `original/<случайный UUID>` не зависит от filename и не попадает в публичный JSON. Сообщение содержит до 10 уникальных UUID; API проверяет существование и принадлежность каждого тому же чату. Идемпотентность сохраняется: ключ блокируется транзакционно в PostgreSQL, отсутствующий список и `[]` эквивалентны, изменение текста или списка возвращает 409.
|
||||||
|
|
||||||
|
До чтения тела проверяются аутентификация, членство, filename, Content-Length ≤10 MiB и заявленный MIME. Тело читается с известным ограничением; chunked upload не поддерживается и возвращает 400. Pillow определяет реальный PNG/JPEG, сверяет MIME, проверяет число пикселей **до декодирования**, затем verify/load выявляют повреждение. Лимит 40 млн пикселей; превышение — 413, чужой/повреждённый формат — 415, опасное имя — 400. Filename с `/`, `\`, NUL, C0/C1/DEL, пустой или длиннее 255 символов отклоняется. Изображение не перекодируется: S3 получает исходные bytes, API возвращает тот же оригинал с правильным MIME, без redirect.
|
||||||
|
|
||||||
|
Сначала синхронный S3 PUT, затем INSERT метаданных и COMMIT PostgreSQL, только после этого 201 `ready`, `variants=[]`, `error_code=null`. При известном отказе до COMMIT выполняется попытка удалить записанный объект. При неясном исходе COMMIT объект не удаляется: это защищает потенциально подтверждённые метаданные. S3 и PostgreSQL не составляют распределённую транзакцию; аварийное завершение/отказ компенсации может оставить orphan. Автоматической сверки нет (это уровень 4/5).
|
||||||
|
|
||||||
|
`postgres_data` хранит БД, `./data/s3` (persistent bind mount) — master/volume/filer. Filer использует LevelDB2 явно в `/data/filerldb2`, а не ephemeral `/tmp`. На API нет тома или локального каталога оригиналов. SeaweedFS master, volume, filer и S3 запущены одним сервисом `s3`. Все его порты и порты БД/API доступны только в Compose-сети; наружу опубликован лишь localhost LB. S3 требует подпись из сгенерированного JSON с identities; анонимного пользователя нет. На уровне 3 API и bootstrap используют одну административную S3 identity.
|
||||||
|
|
||||||
|
`/health/live` не зависит от хранилищ. `/health/ready` требует PostgreSQL SELECT и S3 HeadBucket, поскольку S3 входит в обязательный синхронный путь; HAProxy проверяет readiness, исключая неготовые реплики. Проверка соединения не гарантирует свободное место для будущей записи. S3 имеет таймаут подключения 2s, чтения 3s и одну попытку, без скрытого retry upload.
|
||||||
|
|
||||||
|
`.env.example` соответствует реально читаемым переменным: `HOST_PORT`, `SESSION_TTL_SECONDS`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS`. `DATABASE_URL_FILE`, S3 endpoint/region/bucket и пути credentials задаёт Compose. `APP_HOST`/`APP_PORT` заданы там же. Ограничения 10485760 bytes / 40000000 pixels фиксированы контрактом в коде; Redis/Valkey для оценки 3 не используется, сессии устойчиво хранятся в PostgreSQL.
|
||||||
|
|
||||||
## Доказательства критериев
|
## Доказательства критериев
|
||||||
|
|
||||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
Проверено 13.11.2026 (Europe/Moscow). Тесты используют синтетические данные, bearer tokens не сохраняются.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| Публичный smoke | make test | TODO | TODO |
|
|
||||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
|
||||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
|
||||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
| Критерий | Команда / сценарий | Наблюдаемый результат | Доказательство |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| Контракт и публичный smoke lab4 | `make check validate`, `make test BASE_URL=http://localhost:8084` | PASS; исходные public tests/contracts не изменены | `evidence/checks.txt`, `evidence/final.txt` |
|
||||||
|
| Возможности lab3: идентификация, ACL, Unicode, пагинация, идемпотентность | `student-tests/check.py`, `student-tests/idempotency.py` | PASS; 32 конкурентных повтора через ≥2 процесса создали ровно одно сообщение | `evidence/checks.txt` |
|
||||||
|
| Истечение сессии из lab2 | Одноразовая API с `SESSION_TTL_SECONDS=2`, `student-tests/expiry.py` | Сразу GET 200; через 3s GET и logout 401 | `evidence/expiry.txt` |
|
||||||
|
| Реальные PNG/JPEG, limits, filename, MIME, сообщения с вложениями | `.venv/bin/python student-tests/attachments.py --base-url http://localhost:8084` | Побайтное совпадение PNG/JPEG, 10 MiB / 40M pixels приняты, превышения 413, повреждение/MIME 415, чужой чат 404, retry/conflict корректны | `evidence/checks.txt` |
|
||||||
|
| Разделение БД/S3, приватный bucket | `docker compose exec -T api python - < student-tests/storage.py` | Все строки Attachment имеют S3-оригинал с правильным размером/SHA256; anonymous GET реальных ключей →403 | `evidence/checks.txt`, `evidence/final.txt` |
|
||||||
|
| Устойчивые volumes, прежние сессии и оригинал | `make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh` | PASS; рестарт LB, API×2, PostgreSQL и S3 без удаления данных | `evidence/persistence.txt`, `evidence/containers-before.json`, `evidence/containers-after.json` |
|
||||||
|
| Отказ API во время записи | `make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh` | Две реплики до отказа; 95 успешных записей, 3 переходные ошибки, восстановление 2,44s; потерь/дублей нет | `evidence/failover.txt` |
|
||||||
|
| Upload → остановка именно загрузившей реплики → download другой | `python3 student-tests/attachment_failover.py --base-url http://localhost:8084` | PASS отдельно для обеих реальных реплик; остановленный контейнер оставался выключен во время побайтного сравнения | `evidence/attachment-failover.txt` |
|
||||||
|
| Недоступность настоящего S3 | `python3 student-tests/s3_outage.py --base-url http://localhost:8084` | Live 200; ready/upload/content 503; число Attachment не выросло; после восстановления исходные bytes доступны | `evidence/s3-outage.txt` |
|
||||||
|
| Миграции и отсутствие потери данных при rollback | `docker compose exec -T api python - < student-tests/db_check.py` | Повторные миграции сохраняют counts, включая attachments; KDF/token digests и FK/rollback проверены | `evidence/final.txt` |
|
||||||
|
|
||||||
|
Для проверки S3 outage запросы идут изнутри API-контейнера на `localhost:8080`: LB специально исключает обе неготовые реплики. Порты реплик на host не публикуются. Снимки до/после содержат container ID, StartedAt и типы mounts, без env/credentials. После всех отказов обе API-реплики восстановлены, финальный smoke и проверка всех оригиналов повторены.
|
||||||
|
|
||||||
|
Проверку короткого TTL можно повторить отдельным контейнером, не меняя основной стенд:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker compose run -d --rm --no-deps --name gavrilenko-lab4-expiry -v "$PWD:/work:ro" -e SESSION_TTL_SECONDS=2 api
|
||||||
|
until [ "$(docker inspect --format '{{.State.Health.Status}}' gavrilenko-lab4-expiry)" = healthy ]; do sleep 1; done
|
||||||
|
docker exec gavrilenko-lab4-expiry python /work/student-tests/expiry.py --base-url http://localhost:8080
|
||||||
|
docker stop gavrilenko-lab4-expiry
|
||||||
|
```
|
||||||
|
|
||||||
|
В `.github/workflows/application.yml` добавлен отдельный job: сборка Compose, готовность, smoke, собственные проверки, persistence/failover и остановка. На Gitea Actions отключены; hosted job не запускался, соответствующие команды выполнены локально.
|
||||||
|
|
||||||
## Эксперименты и отказы
|
## Эксперименты и отказы
|
||||||
|
|
||||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
Цель — подтвердить корректность уровня 3, а не максимальную пропускную способность. Нагрузочные исследования, RPS/p95 и ≥10 параллельных upload не заявлены. В `student-tests/idempotency.py` проверяются 32 конкурентных повтора сообщения через две реальные реплики.
|
||||||
|
|
||||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
Во время подготовки образ MinIO оказался недоступен в Docker Hub и quay.io; выбран разрешённый SeaweedFS. Первые запуски выявили заполненный диск Docker: сборка завершалась `no space left on device`, затем S3 отказывался писать при свободном месте менее 1%. Освобождён пересоздаваемый build-cache и неиспользуемый скачиваемый образ sitespeedio; чужие volumes/контейнеры не удалялись. OrbStack затем снова показывал ~245 MiB свободного места внутри динамического Linux-диска при 29 GiB на macOS. Для S3 выбран persistent bind mount `./data/s3`, который видит свободное место macOS; уже созданные данные скопированы при остановленном S3. Минимальную защиту SeaweedFS по свободному месту не отключали. Итоговые результаты ниже относятся к повторным запускам после устранения причины.
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| TODO | TODO | TODO | TODO |
|
|
||||||
|
|
||||||
## Ограничения и следующие шаги
|
## Ограничения и следующие шаги
|
||||||
|
|
||||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
Только оценка 3. Нет reconciler для orphan, отдельной ограниченной S3 identity API, presigned flow, thumbnail/worker, исследования памяти и полного fuzz/security audit. HTTP допустим для localhost с синтетическими данными; удалённая демонстрация требует TLS. `http.server`, соединение БД на запрос и отсутствие ограничения общего числа HTTP threads — учебные упрощения. Одна загрузка ограничена по bytes/pixels, но много параллельных декодирований может превысить лимит памяти API 512 MiB. Истёкшие сессии/старые ключи не чистятся фоновым заданием. PostgreSQL, S3, LB и host — одиночные точки отказа; переживается остановка одной API-реплики и рестарт с сохранением volumes, а не потеря диска или host.
|
||||||
|
|||||||
+113
@@ -0,0 +1,113 @@
|
|||||||
|
name: gavrilenko-lab4
|
||||||
|
services:
|
||||||
|
db:
|
||||||
|
image: postgres:17.6-bookworm
|
||||||
|
environment:
|
||||||
|
POSTGRES_USER: messenger
|
||||||
|
POSTGRES_DB: messenger
|
||||||
|
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||||
|
secrets: [db_password]
|
||||||
|
volumes:
|
||||||
|
- postgres_data:/var/lib/postgresql/data
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1
|
||||||
|
mem_limit: 512m
|
||||||
|
restart: unless-stopped
|
||||||
|
migrate:
|
||||||
|
build: .
|
||||||
|
command: ["python", "src/db.py"]
|
||||||
|
environment:
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
secrets: [database_url]
|
||||||
|
depends_on:
|
||||||
|
db:
|
||||||
|
condition: service_healthy
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 128m
|
||||||
|
api:
|
||||||
|
build: .
|
||||||
|
environment:
|
||||||
|
APP_HOST: 0.0.0.0
|
||||||
|
APP_PORT: 8080
|
||||||
|
DATABASE_URL_FILE: /run/secrets/database_url
|
||||||
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||||
|
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_REGION: us-east-1
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||||
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||||
|
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||||
|
depends_on:
|
||||||
|
bucket:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
migrate:
|
||||||
|
condition: service_completed_successfully
|
||||||
|
deploy:
|
||||||
|
replicas: ${API_REPLICAS:-2}
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 4s
|
||||||
|
retries: 15
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 512m
|
||||||
|
stop_grace_period: 15s
|
||||||
|
restart: unless-stopped
|
||||||
|
s3:
|
||||||
|
image: chrislusf/seaweedfs:4.48
|
||||||
|
command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-master.telemetry=false", "-master.volumeSizeLimitMB=100", "-volume.max=8", "-filer", "-s3", "-s3.port=9000", "-s3.config=/run/secrets/s3_config"]
|
||||||
|
secrets: [s3_config]
|
||||||
|
volumes:
|
||||||
|
- ./data/s3:/data
|
||||||
|
- ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro
|
||||||
|
healthcheck:
|
||||||
|
test: ["CMD-SHELL", "wget -q -O /dev/null http://localhost:9333/cluster/status && nc -z localhost 9000"]
|
||||||
|
interval: 2s
|
||||||
|
timeout: 3s
|
||||||
|
retries: 30
|
||||||
|
cpus: 1
|
||||||
|
mem_limit: 512m
|
||||||
|
restart: unless-stopped
|
||||||
|
bucket:
|
||||||
|
build: .
|
||||||
|
command: ["python", "src/storage.py"]
|
||||||
|
environment:
|
||||||
|
S3_ENDPOINT: http://s3:9000
|
||||||
|
S3_BUCKET: messenger-attachments
|
||||||
|
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||||
|
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||||
|
secrets: [s3_access_key_id, s3_secret_access_key]
|
||||||
|
depends_on:
|
||||||
|
s3:
|
||||||
|
condition: service_healthy
|
||||||
|
mem_limit: 128m
|
||||||
|
lb:
|
||||||
|
image: haproxy:3.2.6-alpine
|
||||||
|
volumes:
|
||||||
|
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||||
|
ports:
|
||||||
|
- "127.0.0.1:${HOST_PORT:-8084}:8080"
|
||||||
|
depends_on:
|
||||||
|
api:
|
||||||
|
condition: service_healthy
|
||||||
|
cpus: 0.5
|
||||||
|
mem_limit: 128m
|
||||||
|
restart: unless-stopped
|
||||||
|
secrets:
|
||||||
|
s3_config:
|
||||||
|
file: ./secrets/s3_config.json
|
||||||
|
s3_access_key_id:
|
||||||
|
file: ./secrets/s3_access_key_id
|
||||||
|
s3_secret_access_key:
|
||||||
|
file: ./secrets/s3_secret_access_key
|
||||||
|
db_password:
|
||||||
|
file: ./secrets/db_password
|
||||||
|
database_url:
|
||||||
|
file: ./secrets/database_url
|
||||||
|
volumes:
|
||||||
|
postgres_data:
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
Container gavrilenko-lab4-api-2 Running
|
||||||
|
Container gavrilenko-lab4-s3-1 Running
|
||||||
|
Container gavrilenko-lab4-lb-1 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Starting
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Starting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Started
|
||||||
|
Container gavrilenko-lab4-migrate-1 Started
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
Container gavrilenko-lab4-api-1 Starting
|
||||||
|
Container gavrilenko-lab4-api-1 Started
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-lb-1 Waiting
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-lb-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-s3-1 Running
|
||||||
|
Container gavrilenko-lab4-lb-1 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Running
|
||||||
|
Container gavrilenko-lab4-api-1 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Starting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Starting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Started
|
||||||
|
Container gavrilenko-lab4-bucket-1 Started
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
Container gavrilenko-lab4-api-2 Starting
|
||||||
|
Container gavrilenko-lab4-api-2 Started
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-lb-1 Waiting
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
Container gavrilenko-lab4-lb-1 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
PASS: upload on 688c19226a30, stopped that container, identical download on f831a9d0e04f
|
||||||
|
PASS: upload on f831a9d0e04f, stopped that container, identical download on 688c19226a30
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
.venv/bin/python scripts/check_contract.py
|
||||||
|
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||||
|
.venv/bin/python scripts/validate_openapi.py
|
||||||
|
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||||
|
python3 tests/smoke.py --base-url "http://localhost:8084"
|
||||||
|
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
PASS: validation, access, Unicode, pagination and cursor scope
|
||||||
|
PASS: malformed JSON and duplicate username
|
||||||
|
PASS: Unicode/colon passwords, length boundaries, invalid chat creation
|
||||||
|
PASS: 32 concurrent retries across >=2 replicas: one message; conflict, access, key scope and validation
|
||||||
|
PASS: PNG/JPEG byte equality, ACL, unsafe filenames, MIME/corruption, 10 MiB and 40M pixel boundaries, attachment-only messages and retries
|
||||||
|
PASS: all persisted attachments match S3 size/SHA256; anonymous GET of every known object returns 403
|
||||||
|
PASS: repeated migrations preserve row counts; scrypt/token digests; FK and transaction rollback
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"service": "api",
|
||||||
|
"id": "688c19226a30",
|
||||||
|
"started_at": "2026-11-12T22:51:57.513275606Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "api",
|
||||||
|
"id": "f831a9d0e04f",
|
||||||
|
"started_at": "2026-11-12T22:52:05.589762117Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "db",
|
||||||
|
"id": "cbe4bf597e11",
|
||||||
|
"started_at": "2026-11-12T22:51:02.256789826Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "lb",
|
||||||
|
"id": "bc38b50d7b1f",
|
||||||
|
"started_at": "2026-11-12T22:51:27.647106401Z"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "s3",
|
||||||
|
"id": "c02a5b24b683",
|
||||||
|
"started_at": "2026-11-12T22:52:25.917451362Z"
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"service": "api",
|
||||||
|
"id": "688c19226a30",
|
||||||
|
"started_at": "2026-11-12T22:48:37.093655957Z",
|
||||||
|
"mounts": [
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/database_url",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/s3_access_key_id",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/s3_secret_access_key",
|
||||||
|
"name": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "api",
|
||||||
|
"id": "f831a9d0e04f",
|
||||||
|
"started_at": "2026-11-12T22:48:37.221951937Z",
|
||||||
|
"mounts": [
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/database_url",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/s3_access_key_id",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/s3_secret_access_key",
|
||||||
|
"name": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "db",
|
||||||
|
"id": "cbe4bf597e11",
|
||||||
|
"started_at": "2026-11-12T22:48:13.070939549Z",
|
||||||
|
"mounts": [
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/db_password",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "volume",
|
||||||
|
"destination": "/var/lib/postgresql/data",
|
||||||
|
"name": "gavrilenko-lab4_postgres_data"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "lb",
|
||||||
|
"id": "bc38b50d7b1f",
|
||||||
|
"started_at": "2026-11-12T22:48:40.367631022Z",
|
||||||
|
"mounts": [
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/usr/local/etc/haproxy/haproxy.cfg",
|
||||||
|
"name": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"service": "s3",
|
||||||
|
"id": "c02a5b24b683",
|
||||||
|
"started_at": "2026-11-12T22:50:11.324300135Z",
|
||||||
|
"mounts": [
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/data",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/etc/seaweedfs/filer.toml",
|
||||||
|
"name": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "bind",
|
||||||
|
"destination": "/run/secrets/s3_config",
|
||||||
|
"name": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
PASS: session expires with TTL=2; expired token cannot read or logout
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8084"
|
||||||
|
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
{"observed_instances_before": 2, "successful_writes": 95, "failed_attempts": 3, "first_recovery_seconds": 2.44}
|
||||||
|
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
python3 tests/smoke.py --base-url "http://localhost:8084"
|
||||||
|
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||||
|
PASS: all persisted attachments match S3 size/SHA256; anonymous GET of every known object returns 403
|
||||||
|
PASS: repeated migrations preserve row counts; scrypt/token digests; FK and transaction rollback
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8084"
|
||||||
|
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||||
|
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
Container gavrilenko-lab4-s3-1 Stopping
|
||||||
|
Container gavrilenko-lab4-s3-1 Stopped
|
||||||
|
PASS: actual S3 stop -> live 200, ready/upload/content 503 with request IDs; no new metadata
|
||||||
|
Container gavrilenko-lab4-lb-1 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Running
|
||||||
|
Container gavrilenko-lab4-api-1 Running
|
||||||
|
Container gavrilenko-lab4-api-2 Running
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Starting
|
||||||
|
Container gavrilenko-lab4-s3-1 Started
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Starting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Started
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Starting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Started
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Waiting
|
||||||
|
Container gavrilenko-lab4-lb-1 Waiting
|
||||||
|
Container gavrilenko-lab4-db-1 Waiting
|
||||||
|
Container gavrilenko-lab4-migrate-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-1 Waiting
|
||||||
|
Container gavrilenko-lab4-api-2 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Waiting
|
||||||
|
Container gavrilenko-lab4-s3-1 Healthy
|
||||||
|
Container gavrilenko-lab4-db-1 Healthy
|
||||||
|
Container gavrilenko-lab4-bucket-1 Exited
|
||||||
|
Container gavrilenko-lab4-lb-1 Healthy
|
||||||
|
Container gavrilenko-lab4-migrate-1 Exited
|
||||||
|
Container gavrilenko-lab4-api-1 Healthy
|
||||||
|
Container gavrilenko-lab4-api-2 Healthy
|
||||||
|
PASS: S3 recovery preserves original bytes
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[leveldb2]
|
||||||
|
enabled = true
|
||||||
|
dir = "/data/filerldb2"
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
global
|
||||||
|
log stdout format raw local0
|
||||||
|
maxconn 256
|
||||||
|
|
||||||
|
defaults
|
||||||
|
mode http
|
||||||
|
log global
|
||||||
|
timeout connect 1s
|
||||||
|
timeout client 10s
|
||||||
|
timeout server 10s
|
||||||
|
timeout check 1s
|
||||||
|
retries 0
|
||||||
|
|
||||||
|
resolvers docker
|
||||||
|
nameserver dns 127.0.0.11:53
|
||||||
|
resolve_retries 3
|
||||||
|
timeout resolve 1s
|
||||||
|
timeout retry 1s
|
||||||
|
hold valid 1s
|
||||||
|
hold obsolete 1s
|
||||||
|
hold nx 1s
|
||||||
|
|
||||||
|
frontend public
|
||||||
|
bind :8080
|
||||||
|
default_backend messenger
|
||||||
|
|
||||||
|
backend messenger
|
||||||
|
balance roundrobin
|
||||||
|
option httpchk GET /health/ready
|
||||||
|
http-check expect status 200
|
||||||
|
server-template api 10 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none
|
||||||
@@ -0,0 +1,33 @@
|
|||||||
|
CREATE TABLE users (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||||
|
display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0),
|
||||||
|
password_salt bytea NOT NULL,
|
||||||
|
password_hash bytea NOT NULL,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE TABLE chats (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||||
|
);
|
||||||
|
CREATE TABLE chat_members (
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
PRIMARY KEY (chat_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
|
||||||
|
CREATE TABLE messages (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
sender_id uuid NOT NULL,
|
||||||
|
text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
|
||||||
|
CREATE TABLE sessions (
|
||||||
|
token_hash bytea PRIMARY KEY,
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
expires_at timestamptz NOT NULL
|
||||||
|
);
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
CREATE TABLE message_keys (
|
||||||
|
user_id uuid NOT NULL REFERENCES users(id),
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
key varchar(128) NOT NULL,
|
||||||
|
message_id uuid NOT NULL REFERENCES messages(id),
|
||||||
|
expires_at timestamptz NOT NULL,
|
||||||
|
PRIMARY KEY (user_id, chat_id, key)
|
||||||
|
);
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
CREATE TABLE attachments (
|
||||||
|
id uuid PRIMARY KEY,
|
||||||
|
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||||
|
owner_id uuid NOT NULL,
|
||||||
|
filename varchar(255) NOT NULL,
|
||||||
|
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||||
|
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||||
|
sha256 char(64) NOT NULL,
|
||||||
|
object_key text NOT NULL UNIQUE,
|
||||||
|
status text NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||||
|
variants jsonb NOT NULL DEFAULT '[]',
|
||||||
|
error_code text DEFAULT NULL CHECK (error_code IS NULL),
|
||||||
|
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||||
|
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members(chat_id, user_id)
|
||||||
|
);
|
||||||
|
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||||
|
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||||
|
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
psycopg[binary]==3.2.10
|
||||||
|
boto3==1.44.2
|
||||||
|
Pillow==13.0.0
|
||||||
Executable
+5
@@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
docker compose stop lb api db s3
|
||||||
|
docker compose up -d --wait
|
||||||
Executable
+35
@@ -0,0 +1,35 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
python3 - <<'PY'
|
||||||
|
import os
|
||||||
|
import json
|
||||||
|
import secrets
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
os.umask(0o077)
|
||||||
|
root = Path('secrets')
|
||||||
|
root.mkdir(exist_ok=True)
|
||||||
|
Path('data/s3').mkdir(parents=True, exist_ok=True)
|
||||||
|
password_file = root / 'db_password'
|
||||||
|
if not password_file.exists():
|
||||||
|
password_file.write_text(secrets.token_hex(32))
|
||||||
|
password = password_file.read_text().strip()
|
||||||
|
url = root / 'database_url'
|
||||||
|
if not url.exists():
|
||||||
|
from urllib.parse import quote
|
||||||
|
url.write_text('postgresql://messenger:' + quote(password, safe='') + '@db:5432/messenger')
|
||||||
|
# Private parent directory protects files on the host; API UID must read its mount.
|
||||||
|
url.chmod(0o644)
|
||||||
|
for name in ('s3_access_key_id', 's3_secret_access_key'):
|
||||||
|
target = root / name
|
||||||
|
if not target.exists():
|
||||||
|
target.write_text(secrets.token_hex(24))
|
||||||
|
target.chmod(0o644)
|
||||||
|
config = root / 's3_config.json'
|
||||||
|
if not config.exists():
|
||||||
|
config.write_text(json.dumps({'identities': [{'name': 'lab-admin', 'credentials': [{'accessKey': (root / 's3_access_key_id').read_text().strip(), 'secretKey': (root / 's3_secret_access_key').read_text().strip()}], 'actions': ['Admin', 'Read', 'Write', 'List', 'Tagging']}]}))
|
||||||
|
if not Path('.env').exists():
|
||||||
|
Path('.env').write_text('HOST_PORT=8084\nSESSION_TTL_SECONDS=3600\n')
|
||||||
|
print('Local secrets and .env ready (existing files preserved)')
|
||||||
|
PY
|
||||||
Executable
+13
@@ -0,0 +1,13 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID from the API response}"
|
||||||
|
for id in $(docker compose ps -q api); do
|
||||||
|
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||||
|
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||||
|
docker stop --time 5 "$id"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
echo 'Target is not a running API replica in this Compose project' >&2
|
||||||
|
exit 1
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
"""One transaction per connection; versioned migrations run as a separate job."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from psycopg.rows import dict_row
|
||||||
|
|
||||||
|
|
||||||
|
def connect():
|
||||||
|
# shortcut: one connection per request; add a bounded pool before load testing.
|
||||||
|
dsn = Path(os.environ['DATABASE_URL_FILE']).read_text().strip()
|
||||||
|
return psycopg.connect(dsn, connect_timeout=3, row_factory=dict_row,
|
||||||
|
options='-c statement_timeout=5000 -c timezone=UTC')
|
||||||
|
|
||||||
|
|
||||||
|
def migrate():
|
||||||
|
with connect() as db:
|
||||||
|
db.execute('SELECT pg_advisory_xact_lock(2303002)')
|
||||||
|
db.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)')
|
||||||
|
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||||
|
if not db.execute('SELECT 1 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone():
|
||||||
|
db.execute(path.read_text())
|
||||||
|
db.execute('INSERT INTO schema_migrations VALUES (%s)', (path.name,))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
migrate()
|
||||||
|
print('Migrations applied')
|
||||||
+349
@@ -0,0 +1,349 @@
|
|||||||
|
"""Shared PostgreSQL state and atomic message retries behind a load balancer."""
|
||||||
|
import base64
|
||||||
|
import binascii
|
||||||
|
import hashlib
|
||||||
|
import hmac
|
||||||
|
import json
|
||||||
|
import io
|
||||||
|
import warnings
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import secrets
|
||||||
|
import signal
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
import psycopg
|
||||||
|
from PIL import Image, UnidentifiedImageError
|
||||||
|
from botocore.exceptions import BotoCoreError, ClientError
|
||||||
|
from storage import s3, BUCKET
|
||||||
|
from db import connect
|
||||||
|
|
||||||
|
INSTANCE_ID = os.environ.get('HOSTNAME') or uuid.uuid4().hex
|
||||||
|
IDEMPOTENCY_TTL = int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400'))
|
||||||
|
if IDEMPOTENCY_TTL < 86400:
|
||||||
|
raise ValueError('IDEMPOTENCY_TTL_SECONDS must be at least 86400')
|
||||||
|
|
||||||
|
MAX_UPLOAD_BYTES = 10485760
|
||||||
|
MAX_IMAGE_PIXELS = 40000000
|
||||||
|
Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS
|
||||||
|
ATTACHMENT_FIELDS = 'id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, variants, error_code, created_at, updated_at'
|
||||||
|
|
||||||
|
USER_FIELDS = 'id, username, display_name, created_at'
|
||||||
|
SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||||
|
if SESSION_TTL <= 0:
|
||||||
|
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||||
|
|
||||||
|
|
||||||
|
class APIError(Exception):
|
||||||
|
def __init__(self, status, code):
|
||||||
|
self.status, self.code = status, code
|
||||||
|
|
||||||
|
|
||||||
|
def require(condition, status=400, code='invalid_request'):
|
||||||
|
if not condition:
|
||||||
|
raise APIError(status, code)
|
||||||
|
|
||||||
|
|
||||||
|
def identifier(value):
|
||||||
|
try:
|
||||||
|
return str(uuid.UUID(value))
|
||||||
|
except (ValueError, TypeError, AttributeError):
|
||||||
|
raise APIError(400, 'invalid_request')
|
||||||
|
|
||||||
|
|
||||||
|
def text(value, maximum):
|
||||||
|
require(isinstance(value, str) and 0 < len(value) <= maximum and bool(value.strip()))
|
||||||
|
try:
|
||||||
|
value.encode('utf-8')
|
||||||
|
except UnicodeEncodeError:
|
||||||
|
raise APIError(400, 'invalid_request')
|
||||||
|
require('\x00' not in value)
|
||||||
|
|
||||||
|
|
||||||
|
def password_hash(password, salt):
|
||||||
|
return hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64)
|
||||||
|
|
||||||
|
|
||||||
|
def serial(value):
|
||||||
|
if isinstance(value, datetime):
|
||||||
|
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||||
|
if isinstance(value, uuid.UUID):
|
||||||
|
return str(value)
|
||||||
|
raise TypeError('Unsupported JSON value')
|
||||||
|
|
||||||
|
|
||||||
|
class Handler(BaseHTTPRequestHandler):
|
||||||
|
def setup(self):
|
||||||
|
super().setup()
|
||||||
|
self.connection.settimeout(5)
|
||||||
|
|
||||||
|
def body(self, fields, optional=()):
|
||||||
|
require(self.headers.get_content_type() == 'application/json', 415, 'unsupported_media_type')
|
||||||
|
require('Transfer-Encoding' not in self.headers)
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get('Content-Length', '0'))
|
||||||
|
require(0 < length <= 65536)
|
||||||
|
value = json.loads(self.rfile.read(length).decode('utf-8'))
|
||||||
|
except (ValueError, UnicodeDecodeError):
|
||||||
|
raise APIError(400, 'invalid_request')
|
||||||
|
require(type(value) is dict and set(fields) <= set(value) <= set(fields) | set(optional))
|
||||||
|
return value
|
||||||
|
|
||||||
|
def credentials(self, scheme):
|
||||||
|
parts = self.headers.get('Authorization', '').split()
|
||||||
|
require(len(parts) == 2 and parts[0].lower() == scheme.lower(), 401, 'unauthorized')
|
||||||
|
return parts[1]
|
||||||
|
|
||||||
|
def chat(self, db, chat_id):
|
||||||
|
row = db.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
|
||||||
|
members = db.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (chat_id,)).fetchall()
|
||||||
|
row['member_ids'] = [x['user_id'] for x in members]
|
||||||
|
return row
|
||||||
|
|
||||||
|
def page(self, db, table, scope, query, where, params):
|
||||||
|
require(set(query) <= {'limit', 'cursor'} and all(len(v) == 1 for v in query.values()))
|
||||||
|
raw = query.get('limit', ['50'])[0]
|
||||||
|
require(re.fullmatch(r'[0-9]{1,3}', raw) is not None)
|
||||||
|
limit = int(raw)
|
||||||
|
require(1 <= limit <= 100)
|
||||||
|
if 'cursor' in query:
|
||||||
|
try:
|
||||||
|
cursor = json.loads(base64.b64decode(query['cursor'][0], altchars=b'-_', validate=True))
|
||||||
|
require(type(cursor) is list and len(cursor) == 3 and cursor[0] == scope)
|
||||||
|
stamp = datetime.fromisoformat(cursor[1].replace('Z', '+00:00'))
|
||||||
|
require(stamp.tzinfo is not None)
|
||||||
|
last_id = identifier(cursor[2])
|
||||||
|
except (ValueError, TypeError, AttributeError, UnicodeError, binascii.Error):
|
||||||
|
raise APIError(400, 'invalid_request')
|
||||||
|
require(db.execute(f'SELECT 1 FROM {table} WHERE {where} AND created_at=%s AND id=%s',
|
||||||
|
(*params, stamp, last_id)).fetchone() is not None)
|
||||||
|
where += ' AND (created_at, id) > (%s, %s)'
|
||||||
|
params = (*params, stamp, last_id)
|
||||||
|
# table and where are internal constants; every client value is a parameter.
|
||||||
|
rows = db.execute(f'SELECT * FROM {table} WHERE {where} ORDER BY created_at, id LIMIT %s',
|
||||||
|
(*params, limit + 1)).fetchall()
|
||||||
|
selected = rows[:limit]
|
||||||
|
cursor = None
|
||||||
|
if len(rows) > limit:
|
||||||
|
last = selected[-1]
|
||||||
|
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']], default=serial).encode()).decode()
|
||||||
|
if table == 'chats':
|
||||||
|
selected = [self.chat(db, row['id']) for row in selected]
|
||||||
|
return {'items': selected, 'next_cursor': cursor}
|
||||||
|
|
||||||
|
def dispatch(self, db, path, query):
|
||||||
|
if self.command == 'GET' and path == '/health/ready':
|
||||||
|
db.execute('SELECT 1')
|
||||||
|
s3.head_bucket(Bucket=BUCKET)
|
||||||
|
return 200, {'status': 'ok'}
|
||||||
|
if self.command == 'POST' and path == '/api/v1/users':
|
||||||
|
body = self.body(('username', 'display_name', 'password'))
|
||||||
|
require(isinstance(body['username'], str) and re.fullmatch(r'[a-z0-9_]{3,32}', body['username']) is not None)
|
||||||
|
text(body['display_name'], 100)
|
||||||
|
text(body['password'], 128)
|
||||||
|
require(len(body['password']) >= 12)
|
||||||
|
salt = secrets.token_bytes(16)
|
||||||
|
user = db.execute(f'INSERT INTO users (id, username, display_name, password_salt, password_hash) VALUES (%s,%s,%s,%s,%s) RETURNING {USER_FIELDS}',
|
||||||
|
(uuid.uuid4(), body['username'], body['display_name'], salt, password_hash(body['password'], salt))).fetchone()
|
||||||
|
return 201, user
|
||||||
|
if self.command == 'POST' and path == '/api/v1/auth/sessions':
|
||||||
|
credential = self.credentials('Basic')
|
||||||
|
try:
|
||||||
|
username, password = base64.b64decode(credential, validate=True).decode('utf-8').split(':', 1)
|
||||||
|
require(re.fullmatch(r'[a-z0-9_]{3,32}', username) is not None and 12 <= len(password) <= 128, 401, 'unauthorized')
|
||||||
|
except (ValueError, UnicodeError, binascii.Error):
|
||||||
|
raise APIError(401, 'unauthorized')
|
||||||
|
require(self.headers.get('Content-Length', '0') == '0' and 'Transfer-Encoding' not in self.headers)
|
||||||
|
user = db.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone()
|
||||||
|
# Run the same KDF for unknown users to avoid the cheap username timing oracle.
|
||||||
|
digest = password_hash(password, user['password_salt'] if user else bytes(16))
|
||||||
|
require(user is not None and hmac.compare_digest(digest, user['password_hash']), 401, 'unauthorized')
|
||||||
|
token = secrets.token_urlsafe(32)
|
||||||
|
session = db.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",
|
||||||
|
(hashlib.sha256(token.encode()).digest(), user['id'], SESSION_TTL)).fetchone()
|
||||||
|
return 201, dict(session, token=token, token_type='Bearer', user={k: user[k] for k in ('id', 'username', 'display_name', 'created_at')})
|
||||||
|
token = self.credentials('Bearer')
|
||||||
|
token_digest = hashlib.sha256(token.encode()).digest()
|
||||||
|
# shortcut: expired rows remain; add scheduled cleanup for a long-lived deployment.
|
||||||
|
actor = db.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', (token_digest,)).fetchone()
|
||||||
|
require(actor is not None, 401, 'unauthorized')
|
||||||
|
actor = actor['user_id']
|
||||||
|
if self.command == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||||
|
db.execute('DELETE FROM sessions WHERE token_hash=%s', (token_digest,))
|
||||||
|
return 204, None
|
||||||
|
if self.command == 'GET' and path == '/api/v1/users/me':
|
||||||
|
return 200, db.execute(f'SELECT {USER_FIELDS} FROM users WHERE id=%s', (actor,)).fetchone()
|
||||||
|
if path == '/api/v1/chats':
|
||||||
|
if self.command == 'POST':
|
||||||
|
body = self.body(('title', 'member_ids'))
|
||||||
|
text(body['title'], 100)
|
||||||
|
members = body['member_ids']
|
||||||
|
require(type(members) is list and 1 <= len(members) <= 100)
|
||||||
|
members = [identifier(x) for x in members]
|
||||||
|
require(len(set(members)) == len(members) and str(actor) in members)
|
||||||
|
existing = db.execute('SELECT id FROM users WHERE id = ANY(%s::uuid[])', (members,)).fetchall()
|
||||||
|
require(len(existing) == len(members))
|
||||||
|
chat_id = uuid.uuid4()
|
||||||
|
db.execute('INSERT INTO chats (id, title) VALUES (%s,%s)', (chat_id, body['title']))
|
||||||
|
with db.cursor() as cur:
|
||||||
|
cur.executemany('INSERT INTO chat_members VALUES (%s,%s)', [(chat_id, member) for member in members])
|
||||||
|
return 201, self.chat(db, chat_id)
|
||||||
|
if self.command == 'GET':
|
||||||
|
return 200, self.page(db, 'chats', 'chats:' + str(actor), query,
|
||||||
|
'id IN (SELECT chat_id FROM chat_members WHERE user_id=%s)', (actor,))
|
||||||
|
attachment = re.fullmatch(r'/api/v1/attachments/([^/]+)(/content)?', path)
|
||||||
|
if attachment and self.command == 'GET':
|
||||||
|
attachment_id = identifier(attachment[1])
|
||||||
|
row = db.execute('SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',
|
||||||
|
(attachment_id, actor)).fetchone()
|
||||||
|
require(row is not None, 404, 'not_found')
|
||||||
|
if not attachment[2]:
|
||||||
|
require(not query)
|
||||||
|
return 200, {k: row[k] for k in ATTACHMENT_FIELDS.split(', ')}
|
||||||
|
require(set(query) <= {'variant'} and all(len(v) == 1 for v in query.values()))
|
||||||
|
require(query.get('variant', ['original']) == ['original'], 404, 'not_found')
|
||||||
|
response = s3.get_object(Bucket=BUCKET, Key=row['object_key'])
|
||||||
|
with response['Body'] as stream:
|
||||||
|
data = stream.read(MAX_UPLOAD_BYTES + 1)
|
||||||
|
require(len(data) == row['size_bytes'] and hashlib.sha256(data).hexdigest() == row['sha256'], 503, 'unavailable')
|
||||||
|
return 200, (data, row['content_type'])
|
||||||
|
match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages|/attachments)?', path)
|
||||||
|
if match:
|
||||||
|
chat_id = identifier(match[1])
|
||||||
|
require(db.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, actor)).fetchone() is not None, 404, 'not_found')
|
||||||
|
if not match[2] and self.command == 'GET':
|
||||||
|
return 200, self.chat(db, chat_id)
|
||||||
|
if match[2] == '/attachments' and self.command == 'POST':
|
||||||
|
require(set(query) == {'filename'} and len(query['filename']) == 1)
|
||||||
|
filename = query['filename'][0]
|
||||||
|
text(filename, 255)
|
||||||
|
require(not any(c in '/\\' or ord(c) < 32 or 127 <= ord(c) <= 159 for c in filename))
|
||||||
|
require('Transfer-Encoding' not in self.headers)
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get('Content-Length', '0'))
|
||||||
|
except ValueError:
|
||||||
|
raise APIError(400, 'invalid_request')
|
||||||
|
require(length > 0)
|
||||||
|
require(length <= MAX_UPLOAD_BYTES, 413, 'payload_too_large')
|
||||||
|
mime = self.headers.get_content_type()
|
||||||
|
require(mime in ('image/png', 'image/jpeg'), 415, 'unsupported_media_type')
|
||||||
|
data = self.rfile.read(length)
|
||||||
|
require(len(data) == length)
|
||||||
|
try:
|
||||||
|
with warnings.catch_warnings():
|
||||||
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||||
|
with Image.open(io.BytesIO(data), formats=('PNG', 'JPEG')) as image:
|
||||||
|
require(image.width * image.height <= MAX_IMAGE_PIXELS, 413, 'payload_too_large')
|
||||||
|
require(Image.MIME[image.format] == mime, 415, 'unsupported_media_type')
|
||||||
|
image.verify()
|
||||||
|
with Image.open(io.BytesIO(data), formats=('PNG', 'JPEG')) as image:
|
||||||
|
image.load()
|
||||||
|
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
|
||||||
|
raise APIError(413, 'payload_too_large')
|
||||||
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||||
|
raise APIError(415, 'unsupported_media_type')
|
||||||
|
attachment_id = uuid.uuid4()
|
||||||
|
object_key = 'original/' + attachment_id.hex
|
||||||
|
# shortcut: process death between S3 PUT and DB commit can leave an orphan; add reconciliation for grade 4.
|
||||||
|
self.uploaded_key = object_key
|
||||||
|
s3.put_object(Bucket=BUCKET, Key=object_key, Body=data, ContentType=mime)
|
||||||
|
row = db.execute(f'INSERT INTO attachments (id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING {ATTACHMENT_FIELDS}',
|
||||||
|
(attachment_id, chat_id, actor, filename, mime, length, hashlib.sha256(data).hexdigest(), object_key)).fetchone()
|
||||||
|
return 201, row
|
||||||
|
if match[2] == '/messages':
|
||||||
|
if self.command == 'GET':
|
||||||
|
return 200, self.page(db, 'messages', 'messages:' + chat_id, query, 'chat_id=%s', (chat_id,))
|
||||||
|
if self.command == 'POST':
|
||||||
|
key = self.headers.get('Idempotency-Key', '')
|
||||||
|
require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None)
|
||||||
|
body = self.body(('text',), ('attachment_ids',))
|
||||||
|
ids = body.get('attachment_ids', [])
|
||||||
|
require(type(ids) is list and len(ids) <= 10)
|
||||||
|
ids = [identifier(x) for x in ids]
|
||||||
|
require(len(set(ids)) == len(ids))
|
||||||
|
require(isinstance(body['text'], str) and len(body['text']) <= 4000)
|
||||||
|
text('x' + body['text'], 4001)
|
||||||
|
require(bool(body['text'].strip()) or bool(ids))
|
||||||
|
found = db.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])', (chat_id, ids)).fetchall()
|
||||||
|
require(len(found) == len(ids), 404, 'not_found')
|
||||||
|
# Serialize the same scope across replicas; lock, effect and key commit together.
|
||||||
|
scope = json.dumps([str(actor), chat_id, key])
|
||||||
|
db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (scope,))
|
||||||
|
db.execute('DELETE FROM message_keys WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at <= clock_timestamp()',
|
||||||
|
(actor, chat_id, key))
|
||||||
|
previous = db.execute('SELECT m.* FROM message_keys k JOIN messages m ON m.id=k.message_id WHERE k.user_id=%s AND k.chat_id=%s AND k.key=%s',
|
||||||
|
(actor, chat_id, key)).fetchone()
|
||||||
|
if previous:
|
||||||
|
require(previous['text'] == body['text'] and [str(x) for x in previous['attachment_ids']] == ids, 409, 'conflict')
|
||||||
|
return 201, previous
|
||||||
|
message = db.execute('INSERT INTO messages (id,chat_id,sender_id,text,attachment_ids) VALUES (%s,%s,%s,%s,%s::uuid[]) RETURNING *',
|
||||||
|
(uuid.uuid4(), chat_id, actor, body['text'], ids)).fetchone()
|
||||||
|
db.execute("INSERT INTO message_keys VALUES (%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')",
|
||||||
|
(actor, chat_id, key, message['id'], IDEMPOTENCY_TTL))
|
||||||
|
return 201, message
|
||||||
|
raise APIError(404, 'not_found')
|
||||||
|
|
||||||
|
def handle_request(self):
|
||||||
|
self.uploaded_key = None
|
||||||
|
request_id = uuid.uuid4().hex
|
||||||
|
path = urlsplit(self.path)
|
||||||
|
try:
|
||||||
|
if self.command == 'GET' and path.path == '/health/live':
|
||||||
|
status, body = 200, {'status': 'ok'}
|
||||||
|
else:
|
||||||
|
with connect() as db:
|
||||||
|
status, body = self.dispatch(db, path.path, parse_qs(path.query, keep_blank_values=True))
|
||||||
|
# COMMIT outcome can be unknown after connection loss; never delete a possibly committed object.
|
||||||
|
self.uploaded_key = None
|
||||||
|
except (BotoCoreError, ClientError):
|
||||||
|
status, body = 503, 'unavailable'
|
||||||
|
except psycopg.errors.UniqueViolation:
|
||||||
|
status, body = 409, 'conflict'
|
||||||
|
except (psycopg.OperationalError, psycopg.errors.QueryCanceled):
|
||||||
|
status, body = 503, 'unavailable'
|
||||||
|
except APIError as exc:
|
||||||
|
status, body = exc.status, exc.code
|
||||||
|
except Exception:
|
||||||
|
status, body = 500, 'internal_error'
|
||||||
|
if self.uploaded_key:
|
||||||
|
try:
|
||||||
|
s3.delete_object(Bucket=BUCKET, Key=self.uploaded_key)
|
||||||
|
except (BotoCoreError, ClientError):
|
||||||
|
print('S3 compensation failed; orphan reconciliation required', flush=True)
|
||||||
|
if status >= 400:
|
||||||
|
body = {'error': {'code': body, 'message': body, 'request_id': request_id}}
|
||||||
|
binary = isinstance(body, tuple)
|
||||||
|
payload = body[0] if binary else b'' if status == 204 else json.dumps(body, ensure_ascii=False, default=serial).encode('utf-8')
|
||||||
|
self.send_response(status)
|
||||||
|
if status != 204:
|
||||||
|
self.send_header('Content-Type', body[1] if binary else 'application/json; charset=utf-8')
|
||||||
|
self.send_header('X-Content-Type-Options', 'nosniff')
|
||||||
|
if status == 401:
|
||||||
|
challenge = 'Basic realm="messenger", charset="UTF-8"' if path.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||||
|
self.send_header('WWW-Authenticate', challenge)
|
||||||
|
self.send_header('Content-Length', str(len(payload)))
|
||||||
|
self.send_header('X-Request-Id', request_id)
|
||||||
|
self.send_header('X-Instance-Id', INSTANCE_ID)
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(payload)
|
||||||
|
|
||||||
|
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = handle_request
|
||||||
|
|
||||||
|
def log_message(self, format, *args):
|
||||||
|
# Paths may contain cursors; log only method and response status.
|
||||||
|
print(self.command, args[1] if len(args) > 1 else '', flush=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# Non-daemon request threads finish their transactions before process exit.
|
||||||
|
with ThreadingHTTPServer((os.getenv('APP_HOST', '0.0.0.0'), int(os.getenv('APP_PORT', '8080'))), Handler) as server:
|
||||||
|
server.daemon_threads = False
|
||||||
|
def stop(signum, frame):
|
||||||
|
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||||
|
signal.signal(signal.SIGTERM, stop)
|
||||||
|
signal.signal(signal.SIGINT, stop)
|
||||||
|
server.serve_forever(poll_interval=0.1)
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""S3 client shared by request threads; bootstrap runs before API starts."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import boto3
|
||||||
|
from botocore.config import Config
|
||||||
|
from botocore.exceptions import ClientError
|
||||||
|
|
||||||
|
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||||
|
s3 = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||||
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||||
|
aws_access_key_id=Path(os.environ['S3_ACCESS_KEY_ID_FILE']).read_text().strip(),
|
||||||
|
aws_secret_access_key=Path(os.environ['S3_SECRET_ACCESS_KEY_FILE']).read_text().strip(),
|
||||||
|
config=Config(connect_timeout=2, read_timeout=3, retries={'total_max_attempts': 1},
|
||||||
|
s3={'addressing_style': 'path'}))
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
s3.head_bucket(Bucket=BUCKET)
|
||||||
|
except ClientError as exc:
|
||||||
|
if exc.response['Error']['Code'] != '404':
|
||||||
|
raise
|
||||||
|
s3.create_bucket(Bucket=BUCKET)
|
||||||
|
# An absent policy is private by default; remove any accidental anonymous grants.
|
||||||
|
try:
|
||||||
|
s3.delete_bucket_policy(Bucket=BUCKET)
|
||||||
|
except ClientError as exc:
|
||||||
|
if exc.response['Error']['Code'] != 'NoSuchBucketPolicy':
|
||||||
|
raise
|
||||||
|
print('Private bucket ready')
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
"""Stop exactly the replica that accepted the upload; repeat for both replicas."""
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, Failure, arguments
|
||||||
|
|
||||||
|
args = arguments('Upload replica failover')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a, b, eve, chat = c.bootstrap()
|
||||||
|
data = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||||||
|
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', 'api'], text=True).split()
|
||||||
|
assert len(ids) == 2
|
||||||
|
for container in ids:
|
||||||
|
target = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip()
|
||||||
|
try:
|
||||||
|
for attempt in range(30):
|
||||||
|
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201,
|
||||||
|
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||||
|
if result['headers']['X-Instance-Id'] == target:
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
raise AssertionError('Upload did not reach target replica')
|
||||||
|
subprocess.run(['scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, check=True, stdout=subprocess.DEVNULL)
|
||||||
|
deadline = time.monotonic() + 10
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
download = c.call('GET', '/api/v1/attachments/' + result['json']['id'] + '/content', actor=b)
|
||||||
|
break
|
||||||
|
except Failure:
|
||||||
|
if time.monotonic() >= deadline:
|
||||||
|
raise
|
||||||
|
time.sleep(0.2)
|
||||||
|
assert download['headers']['X-Instance-Id'] != target
|
||||||
|
assert download['bytes'] == data
|
||||||
|
state = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.Running}}', container], text=True).strip()
|
||||||
|
assert state == 'false'
|
||||||
|
print('PASS: upload on ' + target + ', stopped that container, identical download on ' + download['headers']['X-Instance-Id'])
|
||||||
|
finally:
|
||||||
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait'], check=True)
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
"""Run with .venv/bin/python against the real Compose stack."""
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import struct
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
import zlib
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote
|
||||||
|
from PIL import Image
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, arguments
|
||||||
|
|
||||||
|
args = arguments('Attachment limits, formats, ACL and message retries')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a, b, eve, chat = c.bootstrap()
|
||||||
|
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
|
||||||
|
png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||||||
|
|
||||||
|
def upload(data, status=201, mime='image/png', filename='photo.png', actor=a):
|
||||||
|
return c.call('POST', path + quote(filename, safe=''), status, actor=actor,
|
||||||
|
raw=data, headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None)
|
||||||
|
|
||||||
|
att = upload(png, filename='Фото.png')['json']
|
||||||
|
for actor in (a, b):
|
||||||
|
result = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)
|
||||||
|
assert result['bytes'] == png and result['headers'].get_content_type() == 'image/png'
|
||||||
|
assert hashlib.sha256(result['bytes']).hexdigest() == att['sha256']
|
||||||
|
for suffix in ('', '/content'):
|
||||||
|
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
|
||||||
|
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401)
|
||||||
|
upload(png, 404, actor=eve)
|
||||||
|
for filename in ('../a.png', 'a/b.png', 'a\\b.png', 'a\x00.png', 'a\n.png', 'a\x7f.png', '', 'a' * 256):
|
||||||
|
upload(png, 400, filename=filename)
|
||||||
|
upload(png, 415, mime='image/jpeg')
|
||||||
|
upload(b'not an image', 415)
|
||||||
|
upload(png[:50], 415)
|
||||||
|
upload(b'GIF89a' + b'\0' * 100, 415)
|
||||||
|
upload(png, 415, mime='application/octet-stream')
|
||||||
|
# Header-only request exercises rejection before reading an oversized body.
|
||||||
|
c.call('POST', path + 'huge.png', 413, actor=a, raw=b'',
|
||||||
|
headers={'Content-Type': 'image/png', 'Content-Length': str(10485761)})
|
||||||
|
|
||||||
|
def chunk(name, data):
|
||||||
|
return struct.pack('>I', len(data)) + name + data + struct.pack('>I', zlib.crc32(name + data))
|
||||||
|
|
||||||
|
huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 8000, 5001, 8, 2, 0, 0, 0)) + chunk(b'IDAT', b'') + chunk(b'IEND', b'')
|
||||||
|
upload(huge, 413)
|
||||||
|
# Both valid formats, byte and pixel boundaries; bounded compressed fixtures.
|
||||||
|
for size, fmt, mime in (((40, 20), 'JPEG', 'image/jpeg'), ((8000, 5000), 'PNG', 'image/png')):
|
||||||
|
output = io.BytesIO()
|
||||||
|
Image.new('RGB', size).save(output, format=fmt)
|
||||||
|
data = output.getvalue()
|
||||||
|
item = upload(data, mime=mime)['json']
|
||||||
|
assert c.call('GET', '/api/v1/attachments/' + item['id'] + '/content', actor=b)['bytes'] == data
|
||||||
|
padded = png + bytes(10485760 - len(png))
|
||||||
|
assert upload(padded)['json']['size_bytes'] == 10485760
|
||||||
|
|
||||||
|
message = c.message(a, chat, '', key='attachment-retry', attachments=[att['id']])['json']
|
||||||
|
assert c.message(a, chat, '', key='attachment-retry', attachments=[att['id']])['json'] == message
|
||||||
|
assert message['attachment_ids'] == [att['id']]
|
||||||
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=a,
|
||||||
|
body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': 'attachment-retry'})
|
||||||
|
other = c.call('POST', '/api/v1/chats', 201, actor=a, body={'title': 'other', 'member_ids': [a['id']]}, schema='Chat')['json']
|
||||||
|
for ids, status in (([att['id']], 404), ([str(uuid.uuid4())], 404), ([att['id']] * 2, 400), (['bad'], 400)):
|
||||||
|
c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', status, actor=a,
|
||||||
|
body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||||
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=a)
|
||||||
|
print('PASS: PNG/JPEG byte equality, ACL, unsafe filenames, MIME/corruption, 10 MiB and 40M pixel boundaries, attachment-only messages and retries')
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
"""Additional checks against a running lab4 server."""
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
from pathlib import Path
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, arguments
|
||||||
|
|
||||||
|
args = arguments('Additional validation checks')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a, b, eve, chat = c.bootstrap()
|
||||||
|
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||||
|
c.call('POST', '/api/v1/users', 400, body={'username': 'BAD', 'display_name': 'X'})
|
||||||
|
c.call('POST', '/api/v1/users', 400, body={'username': 'valid_name', 'display_name': ' '})
|
||||||
|
c.call('POST', '/api/v1/users', 400, body=dict(a['registration'], extra=True))
|
||||||
|
c.call('POST', path, 400, actor=a, body={'text': ' '}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||||
|
c.call('POST', path, 400, actor=a, body={'text': 'ok', 'sender_id': b['id']}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||||
|
c.call('GET', path, 404, actor=eve)
|
||||||
|
c.call('GET', path + '?cursor=bad', 400, actor=a)
|
||||||
|
c.call('GET', path + '?limit=1.5', 400, actor=a)
|
||||||
|
c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': str(uuid.uuid4())})
|
||||||
|
for i in range(5):
|
||||||
|
c.message(a, chat, 'Юникод 👋 ' + str(i))
|
||||||
|
p = c.call('GET', path + '?limit=2', actor=b, schema='MessagePage')['json']
|
||||||
|
other = c.call('POST', '/api/v1/chats', 201, actor=a, body={'title': 'other', 'member_ids': [a['id']]}, schema='Chat')['json']
|
||||||
|
c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + p['next_cursor'], 400, actor=a)
|
||||||
|
items = p['items']
|
||||||
|
while p['next_cursor']:
|
||||||
|
p = c.call('GET', path + '?limit=2&cursor=' + p['next_cursor'], actor=b, schema='MessagePage')['json']
|
||||||
|
items += p['items']
|
||||||
|
assert len(items) == len({x['id'] for x in items}) == 5
|
||||||
|
print('PASS: validation, access, Unicode, pagination and cursor scope')
|
||||||
|
c.call('POST', '/api/v1/users', 400, raw=b'{', headers={'Content-Type': 'application/json'})
|
||||||
|
c.call('POST', '/api/v1/users', 409, body=a['registration'])
|
||||||
|
print('PASS: malformed JSON and duplicate username')
|
||||||
|
import base64
|
||||||
|
# All 128 Unicode code points, including colon, must survive KDF without truncation.
|
||||||
|
registration = {'username': 'unicode_' + uuid.uuid4().hex[:12], 'display_name': 'Unicode', 'password': ':' + '🔐' * 127}
|
||||||
|
u = c.call('POST', '/api/v1/users', 201, body=registration, schema='User')['json']
|
||||||
|
credential = base64.b64encode((u['username'] + ':' + registration['password']).encode()).decode()
|
||||||
|
session = c.call('POST', '/api/v1/auth/sessions', 201, headers={'Authorization': 'Basic ' + credential}, schema='Session')['json']
|
||||||
|
assert session['user'] == u
|
||||||
|
c.call('POST', '/api/v1/users', 400, body=dict(registration, password='a' * 11))
|
||||||
|
c.call('POST', '/api/v1/users', 400, body=dict(registration, password='a' * 129))
|
||||||
|
# Creating a chat with a missing participant must leave no visible chat behind.
|
||||||
|
before = c.call('GET', '/api/v1/chats?limit=100', actor=a, schema='ChatPage')['json']
|
||||||
|
for members in ([a['id'], str(uuid.uuid4())], [a['id'], a['id']], [b['id']]):
|
||||||
|
c.call('POST', '/api/v1/chats', 400, actor=a, body={'title': 'invalid', 'member_ids': members})
|
||||||
|
assert c.call('GET', '/api/v1/chats?limit=100', actor=a, schema='ChatPage')['json'] == before
|
||||||
|
print('PASS: Unicode/colon passwords, length boundaries, invalid chat creation')
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
"""Run inside API container: migrations, constraints, rollback and password storage."""
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
sys.path.insert(0, '/app/src')
|
||||||
|
import psycopg
|
||||||
|
from db import connect, migrate
|
||||||
|
|
||||||
|
|
||||||
|
def counts():
|
||||||
|
with connect() as db:
|
||||||
|
return {name: db.execute('SELECT count(*) AS n FROM ' + name).fetchone()['n']
|
||||||
|
for name in ('users', 'chats', 'chat_members', 'messages', 'sessions', 'message_keys', 'attachments', 'schema_migrations')}
|
||||||
|
|
||||||
|
|
||||||
|
before = counts()
|
||||||
|
migrate()
|
||||||
|
migrate()
|
||||||
|
assert counts() == before
|
||||||
|
with connect() as db:
|
||||||
|
assert db.execute('SELECT count(*) AS n FROM users').fetchone()['n'] > 0
|
||||||
|
assert db.execute('SELECT 1 FROM users WHERE octet_length(password_salt) != 16 OR octet_length(password_hash) != 64').fetchone() is None
|
||||||
|
assert db.execute('SELECT 1 FROM sessions WHERE octet_length(token_hash) != 32').fetchone() is None
|
||||||
|
chat_id = uuid.uuid4()
|
||||||
|
try:
|
||||||
|
with connect() as db:
|
||||||
|
db.execute('INSERT INTO chats (id,title) VALUES (%s,%s)', (chat_id, 'rollback check'))
|
||||||
|
db.execute('INSERT INTO chat_members VALUES (%s,%s)', (chat_id, uuid.uuid4()))
|
||||||
|
except psycopg.errors.ForeignKeyViolation:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
raise AssertionError('Foreign key did not reject missing member')
|
||||||
|
with connect() as db:
|
||||||
|
assert db.execute('SELECT 1 FROM chats WHERE id=%s', (chat_id,)).fetchone() is None
|
||||||
|
assert counts() == before
|
||||||
|
print('PASS: repeated migrations preserve row counts; scrypt/token digests; FK and transaction rollback')
|
||||||
@@ -0,0 +1,15 @@
|
|||||||
|
"""Run against an API started with SESSION_TTL_SECONDS=2; tokens never printed."""
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, arguments
|
||||||
|
|
||||||
|
args = arguments('Short-TTL session expiry')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a = c.user('expiry')
|
||||||
|
c.call('GET', '/api/v1/users/me', actor=a, schema='User')
|
||||||
|
time.sleep(3)
|
||||||
|
c.call('GET', '/api/v1/users/me', 401, actor=a)
|
||||||
|
c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=a)
|
||||||
|
print('PASS: session expires with TTL=2; expired token cannot read or logout')
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
"""Concurrent retries through the LB must reach different real API processes."""
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, arguments
|
||||||
|
|
||||||
|
args = arguments('Shared idempotency across replicas')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a, b, eve, chat = c.bootstrap()
|
||||||
|
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||||
|
key = uuid.uuid4().hex
|
||||||
|
|
||||||
|
|
||||||
|
def send(i):
|
||||||
|
client = Client(args.base_url, args.ca_file)
|
||||||
|
body = {'text': 'Один результат 👋'}
|
||||||
|
if i % 2:
|
||||||
|
body['attachment_ids'] = []
|
||||||
|
return client.call('POST', path, 201, actor=a, body=body,
|
||||||
|
headers={'Idempotency-Key': key}, schema='Message')
|
||||||
|
|
||||||
|
|
||||||
|
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||||
|
responses = list(pool.map(send, range(32)))
|
||||||
|
assert len({r['headers']['X-Instance-Id'] for r in responses}) >= 2
|
||||||
|
assert all(r['json'] == responses[0]['json'] for r in responses)
|
||||||
|
page = c.call('GET', path, actor=b, schema='MessagePage')['json']
|
||||||
|
assert page['items'] == [responses[0]['json']]
|
||||||
|
c.call('POST', path, 409, actor=a, body={'text': 'Другой текст'}, headers={'Idempotency-Key': key})
|
||||||
|
c.call('POST', path, 404, actor=eve, body={'text': 'Один результат 👋'}, headers={'Idempotency-Key': key})
|
||||||
|
other = c.message(b, chat, 'Другой пользователь', key=key)['json']
|
||||||
|
assert other['id'] != responses[0]['json']['id']
|
||||||
|
second_chat = c.call('POST', '/api/v1/chats', 201, actor=a,
|
||||||
|
body={'title': 'Другой чат', 'member_ids': [a['id']]}, schema='Chat')['json']
|
||||||
|
assert c.message(a, second_chat, 'Другой чат', key=key)['json']['id'] != other['id']
|
||||||
|
for bad_key in ('', 'has spaces', 'x' * 129):
|
||||||
|
c.call('POST', path, 400, actor=a, body={'text': 'Invalid key'}, headers={'Idempotency-Key': bad_key})
|
||||||
|
print('PASS: 32 concurrent retries across >=2 replicas: one message; conflict, access, key scope and validation')
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
"""Stop the actual S3 container, probe API directly internally, then restore S3."""
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||||
|
from client import Client, arguments
|
||||||
|
|
||||||
|
args = arguments('Real S3 outage')
|
||||||
|
c = Client(args.base_url, args.ca_file)
|
||||||
|
a, b, eve, chat = c.bootstrap()
|
||||||
|
att, data = c.attachment(a, chat)
|
||||||
|
probe = r'''
|
||||||
|
import json, sys, urllib.request, urllib.error
|
||||||
|
sys.path.insert(0, '/app/src')
|
||||||
|
from db import connect
|
||||||
|
info = json.load(sys.stdin)
|
||||||
|
def call(method, path, expected, data=None, mime=None):
|
||||||
|
headers = {'Authorization': 'Bearer ' + info['token']}
|
||||||
|
if mime: headers['Content-Type'] = mime
|
||||||
|
request = urllib.request.Request('http://localhost:8080' + path, method=method, data=data, headers=headers)
|
||||||
|
try: response = urllib.request.urlopen(request, timeout=10)
|
||||||
|
except urllib.error.HTTPError as exc: response = exc
|
||||||
|
with response:
|
||||||
|
body = json.load(response)
|
||||||
|
assert response.code == expected, (path, response.code)
|
||||||
|
assert response.headers['X-Request-Id']
|
||||||
|
if expected >= 400:
|
||||||
|
assert body['error']['code'] == 'unavailable'
|
||||||
|
assert body['error']['request_id'] == response.headers['X-Request-Id']
|
||||||
|
with connect() as db:
|
||||||
|
before = db.execute('SELECT count(*) AS n FROM attachments').fetchone()['n']
|
||||||
|
call('GET', '/health/live', 200)
|
||||||
|
call('GET', '/health/ready', 503)
|
||||||
|
call('GET', '/api/v1/attachments/' + info['attachment'] + '/content', 503)
|
||||||
|
# Small valid PNG copied from test fixture through stdin, no host path mount.
|
||||||
|
import base64
|
||||||
|
call('POST', '/api/v1/chats/' + info['chat'] + '/attachments?filename=outage.png', 503,
|
||||||
|
base64.b64decode(info['data']), 'image/png')
|
||||||
|
with connect() as db:
|
||||||
|
assert db.execute('SELECT count(*) AS n FROM attachments').fetchone()['n'] == before
|
||||||
|
print('PASS: actual S3 stop -> live 200, ready/upload/content 503 with request IDs; no new metadata')
|
||||||
|
'''
|
||||||
|
import base64
|
||||||
|
try:
|
||||||
|
subprocess.run(['docker', 'compose', 'stop', 's3'], check=True)
|
||||||
|
subprocess.run(['docker', 'compose', 'exec', '-T', '--index', '1', 'api', 'python', '-c', probe],
|
||||||
|
input=json.dumps({'token': a['token'], 'chat': chat['id'], 'attachment': att['id'], 'data': base64.b64encode(data).decode()}).encode(), check=True)
|
||||||
|
finally:
|
||||||
|
subprocess.run(['docker', 'compose', 'up', '-d', '--wait'], check=True)
|
||||||
|
assert c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)['bytes'] == data
|
||||||
|
print('PASS: S3 recovery preserves original bytes')
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""Run in an API container: python - < student-tests/storage.py."""
|
||||||
|
import hashlib
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
sys.path.insert(0, '/app/src')
|
||||||
|
from db import connect
|
||||||
|
from storage import s3, BUCKET
|
||||||
|
|
||||||
|
with connect() as db:
|
||||||
|
rows = db.execute('SELECT object_key, size_bytes, sha256 FROM attachments').fetchall()
|
||||||
|
assert rows
|
||||||
|
for row in rows:
|
||||||
|
response = s3.get_object(Bucket=BUCKET, Key=row['object_key'])
|
||||||
|
with response['Body'] as stream:
|
||||||
|
data = stream.read(10485761)
|
||||||
|
assert len(data) == row['size_bytes'] and hashlib.sha256(data).hexdigest() == row['sha256']
|
||||||
|
try:
|
||||||
|
urllib.request.urlopen(s3.meta.endpoint_url + '/' + BUCKET + '/' + row['object_key'], timeout=5)
|
||||||
|
except urllib.error.HTTPError as exc:
|
||||||
|
assert exc.code == 403
|
||||||
|
else:
|
||||||
|
raise AssertionError('Anonymous download succeeded')
|
||||||
|
print('PASS: all persisted attachments match S3 size/SHA256; anonymous GET of every known object returns 403')
|
||||||
Reference in New Issue
Block a user