ЛР 4 — БВТ2301 — Чиняева — на 5 #90

Closed
aliror wants to merge 4 commits from aliror/mtusi2026lab4-Chinyaeva:solution/BVT2301-Chinyaeva into main
70 changed files with 5255 additions and 31 deletions
+26 -9
View File
@@ -1,20 +1,37 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Replace every change-me value. The repository contains no real secrets.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
DATABASE_URL=postgresql://messenger:change-me-db-password@postgres:5432/messenger
POSTGRES_DB=messenger
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-db-password
CACHE_URL=redis://redis:6379/0
API_REPLICAS=2
API_PORT=8080
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# Private S3-compatible store (SeaweedFS 3.80).
S3_ENDPOINT=http://seaweedfs:8333
# Public endpoint used only to sign direct-upload URLs for clients.
S3_PUBLIC_ENDPOINT=http://localhost:9000
S3_PORT=9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Restricted identity used by API replicas: Read/Write/List only.
S3_ACCESS_KEY_ID=change-me-app-access
S3_SECRET_ACCESS_KEY=change-me-app-secret
# Administrative identity used only by the one-shot s3-init service.
S3_ADMIN_ACCESS_KEY_ID=change-me-admin-access
S3_ADMIN_SECRET_ACCESS_KEY=change-me-admin-secret
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
MAX_DECODE_CONCURRENCY=1
PRESIGN_TTL_SECONDS=300
PRESIGN_GRACE_SECONDS=3600
ORPHAN_GRACE_SECONDS=86400
S3_CONNECT_TIMEOUT_SECONDS=3
S3_READ_TIMEOUT_SECONDS=30
+8
View File
@@ -0,0 +1,8 @@
# Shell entrypoints are mounted/run inside Linux containers: force LF.
*.sh text eol=lf
infra/** text eol=lf
Dockerfile text eol=lf
*.sql text eol=lf
# Windows host helpers keep native CRLF.
*.cmd text eol=crlf
*.ps1 text eol=crlf
+38
View File
@@ -0,0 +1,38 @@
# Runtime image for the lab3 API. The base tag is pinned; dependencies are
# installed from requirements.txt, never from a mutable latest tag.
#
# Every replica runs exactly the same image. The replica alias is its
# container hostname (see app/config.py), so no container_name is needed and
# Compose can scale the service freely.
FROM python:3.12.7-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
COPY requirements.txt ./
RUN python -m pip install --requirement requirements.txt
COPY app ./app
COPY migrations ./migrations
COPY student-tests ./student-tests
USER app
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=6 \
CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"]
# One worker with a bounded thread pool per replica: the unit of horizontal
# scaling is the container, which keeps instance ids and CPU limits legible.
# Gunicorn stops accepting on SIGTERM and drains in-flight requests for at most
# 8 seconds; Docker's stop_grace_period gives it room to finish.
STOPSIGNAL SIGTERM
CMD ["gunicorn", "--bind=0.0.0.0:8080", "--workers=1", "--threads=8", "--timeout=30", "--graceful-timeout=8", "--keep-alive=5", "--worker-tmp-dir=/dev/shm", "--access-logfile=-", "--error-logfile=-", "app.main:app"]
+57 -4
View File
@@ -4,10 +4,16 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate up down migrate logs test load test-persistence test-failover test-instance test-idempotency test-student test-attachments test-negative test-memory test-presigned test-transfer test-reconcile test-private reconcile rolling-update
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API"
@echo "up/down: local compose stack; migrate: run the one-shot migrator explicitly"
@echo "test-persistence ACTION_SCRIPT=scripts/restart.cmd: restart with volumes"
@echo "test-failover ACTION_SCRIPT=scripts/stop-one.cmd: stop one API replica"
@echo "test-attachments/test-negative/test-memory/test-presigned: lab4 evidence"
@echo "test-reconcile/test-private: S3 orphan and private-bucket checks"
@echo "reconcile: dry-run S3/PostgreSQL reconciler in the tools profile"
check:
$(PYTHON) scripts/check_contract.py
@@ -15,6 +21,18 @@ check:
validate:
$(PYTHON) scripts/validate_openapi.py
up:
docker compose --env-file .env up -d --build
down:
docker compose --env-file .env down
migrate:
docker compose --env-file .env run --rm migrate
logs:
docker compose --env-file .env logs --tail=200
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
@@ -22,9 +40,44 @@ load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-instance:
$(PYTHON) student-tests/instance_spread.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
$(PYTHON) student-tests/idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-attachments:
$(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-negative: test-attachments
test-memory:
$(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-presigned:
$(PYTHON) student-tests/presigned_flow.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-transfer:
$(PYTHON) student-tests/transfer_compare.py --base-url "$(BASE_URL)" --output evidence/transfer-compare.json $(TLS_ARGS)
test-reconcile:
$(PYTHON) student-tests/reconcile_orphans.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-private:
$(PYTHON) student-tests/private_bucket.py --base-url "$(BASE_URL)" $(TLS_ARGS)
reconcile:
docker compose --env-file .env --profile tools run --rm reconcile python -m app.reconcile --dry-run
test-student: test-instance test-idempotency test-attachments test-presigned test-transfer
rolling-update:
docker compose --env-file .env build api
sh scripts/rolling-update.sh
+202 -15
View File
@@ -1,37 +1,224 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Чиняева, БВТ2301
- Целевая оценка: **5**; выбранный трек 5: **Трек 1. Прямой upload (presigned PUT)**
- Commit/PR: коммит реализации `f39ae06`; MR `!90` в `mtusi2026/mtusi2026lab4` (ветка `solution/BVT2301-Chinyaeva`)
- Использованные источники, библиотеки и помощь ИИ: контракт и публичные тесты репозитория; SeaweedFS 3.80 (S3 API), PostgreSQL 16.4, Redis 7.4, nginx 1.27.5, Flask 3.1, boto3 1.35.76, Pillow 11.0.0. ИИ использован как помощник при написании кода, тестов и текста; все команды и результаты ниже воспроизведены локально.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требования: Docker Desktop/Engine с Compose v2, Python 3.9+ для тестов на хосте,
свободные порты `8080` (API через LB) и `9000` (S3 API для presigned upload).
Версии образов закреплены в `compose.yaml`: `postgres:16.4-alpine`,
`redis:7.4.0-alpine`, `chrislusf/seaweedfs:3.80`, `nginx:1.27.5-alpine`.
```sh
git clone <fork-url> mtusi2026lab4-Chinyaeva
cd mtusi2026lab4-Chinyaeva
cp .env.example .env
# Замените все change-me значения в .env. Секреты в Git не попадают.
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
# Дождитесь status=healthy у api и seaweedfs:
docker compose --env-file .env ps
curl -fsS http://localhost:8080/health/ready
python tests/smoke.py --base-url http://localhost:8080
```
Остановка без удаления данных:
```sh
docker compose --env-file .env down
```
Намеренное удаление только учебных данных (необратимо):
```sh
docker compose --env-file .env down -v
```
Секреты создаются локально в `.env` (файл в `.gitignore`). В репозитории нет
паролей, ключей, токенов и подписанных URL. S3 credentials генерируются
оператором; API получает ограниченную identity, а административная identity
используется только одноразовым сервисом `s3-init`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
клиент ── HTTP :8080 ── nginx LB ── API-1 ─┐
API-2 ─┤── PostgreSQL (метаданные)
├── Redis (сессии)
└── SeaweedFS S3 (байты, private bucket)
клиент ── presigned PUT :9000 ─────────────┘ (только трек 1, optional)
```
- **Источник истины:** PostgreSQL для пользователей, чатов, сообщений и
метаданных вложений; Redis — только активные сессии; S3 — только байты.
Локального постоянного каталога с оригиналами у API нет: временный файл
загрузки удаляется в `finally`.
- **LB:** nginx 1.27.5 с `resolver 127.0.0.11` и `server api:8080 resolve`.
Он видит все реплики Compose-сервиса `api`, пассивно исключает недоступную
реплику и повторяет только безопасные ошибки соединения/502/504.
- **Реплики:** две и более, общий PostgreSQL и Redis, `X-Instance-Id` в каждом
ответе. Масштабирование: `API_REPLICAS=3 docker compose ... up -d --scale api=3`.
- **S3:** SeaweedFS 3.80, master/volume/filer/S3 в одном контейнере. Bucket
`messenger-attachments` создаётся идемпотентным `s3-init`; bucket приватный.
API использует identity с `Read/Write/List`, без `Admin`; `s3-init` — с
`Admin`. Порт S3 опубликован только для presigned PUT; master/filer UI
(9333/8888) не публикуются.
- **Согласованность:** proxy upload сначала пишет объект в S3, затем строку в
PostgreSQL. Если COMMIT не удался, объект удаляется компенсацией; если процесс
упал между шагами, объект остаётся orphan и удаляется reconciler'ом после
grace period. Presigned upload создаёт `pending`-строку, а `complete`
проверяет владельца, размер и SHA-256 и только затем переводит в `ready`.
- **Совместимость:** обязательный контракт `contracts/openapi.json` не изменён.
Новые endpoint'ы описаны в `contracts/extensions.openapi.json` и
`contracts/extensions.md`; proxy endpoint остаётся обязательным fallback.
### Переменные `.env`
`DATABASE_URL`, `CACHE_URL`, `S3_ENDPOINT` (внутренний), `S3_PUBLIC_ENDPOINT`
(для подписи), `S3_BUCKET`, `S3_ACCESS_KEY_ID`/`S3_SECRET_ACCESS_KEY`
(ограниченная identity), `S3_ADMIN_*` (только `s3-init`), `MAX_UPLOAD_BYTES`,
`MAX_IMAGE_PIXELS`, `MAX_DECODE_CONCURRENCY`, `PRESIGN_TTL_SECONDS`,
`PRESIGN_GRACE_SECONDS`, `ORPHAN_GRACE_SECONDS`. Все они проброшены в
`compose.yaml`; `*_FILE`-варианты поддерживаются в `app/config.py`.
## Обработка вложений и лимиты
- Принимаются только `image/png` и `image/jpeg`; MIME сверяется с реальным
форматом, а не только с заголовком. Несовпадение/повреждение — `415`.
- Тело читается чанками по 64 KiB во временный файл; лимит 10 485 760 байт
проверяется и по `Content-Length`, и в потоке. Превышение — `413`.
- `Image.open` сначала проверяет размеры; `width * height > 40 000 000` —
`413` до полного декодирования. Полный `verify()`/`load()` ограничен
семафором `MAX_DECODE_CONCURRENCY` (по умолчанию 1 на реплику).
- `filename` — только display-метаданные: запрещены `/`, `\`, NUL, control
characters, `.`/`..`; object key — `chats/<chat_id>/<attachment_uuid>`.
- Скачивание идёт потоком через API с `Content-Type` из метаданных; redirect
на S3 нет. Посторонний участник получает `404` и для метаданных, и для bytes.
- Сообщение с `attachment_ids` проверяет, что каждое вложение существует,
принадлежит этому чату и имеет `status=ready`; чужой чат/неизвестный id —
`404`, неготовое — `409`. Пустой `text` разрешён только при наличии вложений.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8080` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` |
| Структура/контракт не сломаны | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | `evidence/check.txt` |
| S3 в общем стенде, метаданные в PG, байты в private bucket | `docker compose ps`; `private_bucket.py` | SeaweedFS healthy; anonymous GET 403; API identity не может create/delete bucket | `evidence/compose-ps.txt`, `evidence/private-bucket.txt` |
| POST вложения → 201 ready; оригинал побайтно; message с attachment_ids | `tests/smoke.py`, `presigned_flow.py` | SHA-256 и Content-Type совпадают; участник чата читает, посторонний — 404 | `evidence/smoke.txt`, `evidence/presigned-flow.txt` |
| PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому; опасные filename | `student-tests/attachments_negative.py` | 413/415/400 на все негативные случаи, без 500 | `evidence/attachments-negative.txt` |
| Upload → остановка обслужившей реплики → download через другую | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | 2 реплики до отказа; 89 успешных записей; recovery 3.667 s; данные и сессия сохранены | `evidence/failover.txt` |
| Restart с volumes сохраняет метаданные и оригинал | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | PASS; attachment скачан после restart | `evidence/persistence.txt` |
| Уровень 4: тесты слишком большого файла, повреждения, MIME, чужого id, другого чата | `attachments_negative.py` | все сценарии PASS | `evidence/attachments-negative.txt` |
| Уровень 4: компенсация/сверка/очистка с grace period | `student-tests/reconcile_orphans.py` | dry-run видит orphan; свежий объект не удаляется; `--apply` удаляет; stale pending удаляется | `evidence/reconcile.txt` |
| Уровень 4: ≥10 одновременных загрузок, пик RAM | `student-tests/upload_memory.py --workers 10` | 10/10 успешно; рост ~22.5 MiB на реплику при файле 4.43 MB | `evidence/upload-memory.json`, `evidence/upload-memory.txt` |
| Уровень 4: идемпотентный bootstrap, отдельные credentials, закрытая admin-консоль | `private_bucket.py`; повторный `docker compose run --rm s3-init` | повторный bootstrap видит bucket; API identity без Admin; опубликован только S3 API | `evidence/private-bucket.txt`, `evidence/s3-init-idempotent.txt`, `evidence/compose-ps.txt` |
| Трек 1: presigned upload, короткая подпись, unique key, ownership, size/checksum, incomplete | `student-tests/presigned_flow.py` | 201 presign, 200 PUT, 200 complete; wrong checksum/size 400; foreign complete 404; incomplete 409 | `evidence/presigned-flow.txt` |
| Трек 1: сравнение трафика/RAM API на ≥2 размерах | `student-tests/transfer_compare.py` | proxy ≈ 2× file bytes через API; direct ≈ 1× (только verification read); RAM не растёт линейно | `evidence/transfer-compare.json`, `evidence/transfer-compare.txt` |
| Общая репликация/идемпотентность lab3 сохранены | `instance_spread.py`, `idempotency_race.py` | ≥2 реплики; 20/20 replay → один Message | `evidence/instance-spread.txt`, `evidence/idempotency-race.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Эксперименты
## Эксперименты и отказы
### 1. 10 одновременных proxy-загрузок
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Команда:
```sh
python student-tests/upload_memory.py --base-url http://localhost:8080 \
--workers 10 --output evidence/upload-memory.json
```
Файл: синтетический PNG 1400×1000, `4 431 995` байт, 10 параллельных
загрузок. API-контейнеры ограничены 512 MiB. Результат:
```json
{"workers":10,"file_bytes":4431995,"elapsed_seconds":1.697,
"baseline_mib":{"mtusi-lab4-api-1":83.17,"mtusi-lab4-api-2":85.38},
"peak_mib":{"mtusi-lab4-api-1":105.7,"mtusi-lab4-api-2":107.9},
"peak_growth_mib":{"mtusi-lab4-api-1":22.53,"mtusi-lab4-api-2":22.52},
"successful_uploads":10}
```
Пик не растёт как 10×размер файла: тело на диске, декодирование ограничено
семафором, поэтому рост ~22 MiB на реплику. Это измерение одного хоста; оно не
доказывает отсутствие OOM при произвольном размере, но подтверждает заявленную
границу для 10 MiB и 40 млн пикселей.
### 2. Proxy vs presigned direct (трек 1)
Команда:
```sh
python student-tests/transfer_compare.py --base-url http://localhost:8080 \
--output evidence/transfer-compare.json
```
Измерялись суммарные RX+TX байты API-контейнеров через `/proc/net/dev` и пик
памяти через `docker stats`. Результат:
| Размер файла | Proxy: API network | Direct: API network | Proxy: client→API | Direct: client→S3 |
| --- | ---: | ---: | ---: | ---: |
| 552 409 B | 1 120 316 B | 564 368 B | 552 409 B | 552 409 B |
| 4 424 547 B | 8 888 758 B | 4 446 591 B | 4 424 547 B | 4 424 547 B |
Proxy проводит тело через API дважды (вход от клиента и выход в S3); direct
освобождает публичный ingress API, но `complete` один раз читает объект из S3
для проверки SHA-256. Поэтому direct ≈ 1× file bytes, proxy ≈ 2×. RAM API при
direct не растёт: объект читается потоком, без буферизации всего файла
(`api_memory_growth_mib=0.0` в обоих размерах). Это честная цена проверки
целостности; без неё direct был бы ещё дешевле, но weaker guarantee.
### 3. Нагрузка сообщений (регрессия lab3)
```sh
python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
```
30.07 s, 4 closed-loop воркера, 6 789 успешных запросов, 0 ошибок,
225.78 successful RPS, p50 14.8 ms, p95 33.8 ms, p99 66.7 ms. Это не
пропускная способность production: closed-loop и один Docker-host.
## Отказы
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановка одной API-реплики (`stop-one.cmd`) | nginx исключает её, записи идут на вторую; сессия жива | 2 реплики до отказа, 89 успешных записей, 3 неуверенных retry, `first_recovery_seconds=3.667` | ~3.7 s; подтверждённые сообщения и сессия сохранены |
| `docker compose restart` (PostgreSQL, Redis, SeaweedFS, API, LB) | volumes сохраняют данные; API снова ready | persistence PASS; attachment скачан после restart | данные не потеряны; API поднялся после прогрева SeaweedFS |
| S3 недоступен | `/health/ready` = 503, API не принимает новые вложения | при старте API health был 503 до готовности S3; после готовности 200 | зависит от S3; metadata не теряется |
| Обрыв между S3 PUT и INSERT | объект не должен остаться навсегда | proxy-компенсация удаляет объект; reconciler удаляет orphan после grace period | потерян только незавершённый объект; метаданных нет |
| Незавершённый presigned upload | pending не должен жить вечно | reconciler с `--pending-grace-seconds 0` удаляет строку и объект; повторный GET = 404 | потерян только незавершённый upload |
| Чужой/неизвестный attachment_id | 404, не 500 | negative suite PASS | нет |
| Слишком большой/битый файл, MIME spoof | 413/415, не 500 | negative suite PASS | нет |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host не даёт отказоустойчивости к потере host; nginx, PostgreSQL,
Redis и SeaweedFS остаются single points of failure в учебном стенде.
- `complete` читает объект из S3 для проверки SHA-256. Это осознанный
компромисс: direct upload экономит публичный ingress API, но не убирает
verification egress. В production можно использовать S3 checksum API или
доверенный client checksum с отдельной моделью угроз.
- `reconcile` — операторский инструмент, не фоновый демон. В lab5 его можно
запускать по cron/worker с метриками; сейчас он запускается вручную и
поддерживает dry-run/повторный apply.
- Presigned URL публикует S3 API на `localhost:9000`; в другом окружении
`S3_PUBLIC_ENDPOINT` нужно заменить на реальный публичный адрес. Bucket
остаётся приватным, но сетевой доступ к S3 API должен быть ограничен
firewall'ом/прокси в production.
- Лимит 40 млн пикселей и 10 MiB — контрактные; при их увеличении нужно
пересмотреть `MAX_DECODE_CONCURRENCY` и memory limit API.
- Публичные тесты не заменяют защиту: сценарии ниже проверены вручную и
студенческими тестами, но оценка выставляется преподавателем.
## Источники и ИИ
- `contracts/openapi.json`, `contracts/README.md`, `tests/README.md` — контракт
и публичные проверки.
- SeaweedFS S3 configuration and presigned URLs; PostgreSQL deferrable
constraints; boto3 `generate_presigned_url`; Pillow decompression limits.
- ИИ-помощник использован для генерации черновиков кода/тестов и текста
отчёта. Все результаты в `evidence/` получены запуском команд на локальном
стенде; секреты и токены в отчёт не включены.
+1
View File
@@ -0,0 +1 @@
# Messenger API for MTUSI high-load course, laboratory 2.
+135
View File
@@ -0,0 +1,135 @@
"""Attachment upload validation and serialisation.
Design rules for laboratory 4:
* raw bytes are streamed to an ephemeral file, never accumulated in RAM;
* the declared MIME type must match the format sniffed from the bytes;
* the pixel budget is checked before the image is decoded;
* a small semaphore bounds how many full decodes run at once per API replica;
* the object key is generated by the server and never contains the filename.
"""
from __future__ import annotations
import hashlib
import os
import tempfile
import threading
import uuid
from datetime import datetime, timezone
from PIL import Image, UnidentifiedImageError
from .config import settings
from .errors import ApiError, payload_too_large, unsupported_media_type
# Pillow's own bomb guard is a second line of defence; the explicit product
# check below is the contract limit and runs before verify()/load().
Image.MAX_IMAGE_PIXELS = settings.max_image_pixels
_decoders = threading.BoundedSemaphore(settings.max_decode_concurrency)
ALLOWED = {"PNG": "image/png", "JPEG": "image/jpeg"}
def format_time(value: datetime) -> str:
return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def object_key(chat_id: uuid.UUID, attachment_id: uuid.UUID) -> str:
# Random UUIDs make the key unguessable; the client filename is display
# metadata only. A stable prefix helps the reconciler and operators.
return f"chats/{chat_id}/{attachment_id}"
def attachment_json(row: dict) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": int(row["size_bytes"]),
"sha256": row["sha256"],
"status": row["status"],
"variants": [],
"error_code": row["error_code"],
"created_at": format_time(row["created_at"]),
"updated_at": format_time(row["updated_at"]),
}
def read_to_temp(stream) -> tuple[str, int, str]:
"""Stream the request body to a bounded temporary file.
The caller owns the returned path and must delete it in a finally block.
"""
digest = hashlib.sha256()
total = 0
handle = tempfile.NamedTemporaryFile(
mode="wb", prefix="upload-", suffix=".bin", dir=settings.upload_temp_dir, delete=False
)
path = handle.name
try:
while True:
try:
chunk = stream.read(64 * 1024)
except Exception as error:
# Flask raises RequestEntityTooLarge when MAX_CONTENT_LENGTH is
# exceeded on a chunked request.
if getattr(error, "code", None) == 413:
raise payload_too_large() from error
raise
if not chunk:
break
total += len(chunk)
if total > settings.max_upload_bytes:
raise payload_too_large("Файл больше 10 MiB")
digest.update(chunk)
handle.write(chunk)
finally:
handle.close()
if total == 0:
os.unlink(path)
raise unsupported_media_type("Пустой файл")
return path, total, digest.hexdigest()
def validate_image(path: str, declared_content_type: str) -> None:
"""Sniff the real format, enforce the pixel budget, then fully decode."""
try:
with Image.open(path) as image:
actual = ALLOWED.get(image.format or "")
if actual is None:
raise unsupported_media_type("Поддерживаются только PNG и JPEG")
if actual != declared_content_type:
raise unsupported_media_type("Content-Type не совпадает с содержимым")
width, height = image.size
if width < 1 or height < 1:
raise unsupported_media_type("Некорректные размеры изображения")
if width * height > settings.max_image_pixels:
raise payload_too_large("Слишком много пикселей")
# verify() parses the container and checks CRCs without keeping the
# decoded bitmap; load() below is the bounded full decode.
image.verify()
except ApiError:
raise
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error:
raise unsupported_media_type("Повреждённое изображение") from error
with _decoders:
try:
with Image.open(path) as image:
image.load()
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error:
raise unsupported_media_type("Повреждённое изображение") from error
def cleanup(path: str | None) -> None:
if not path:
return
try:
os.unlink(path)
except FileNotFoundError:
return
except OSError:
return
+134
View File
@@ -0,0 +1,134 @@
"""Runtime configuration.
The application never embeds credentials. Local Compose passes environment
variables from an ignored .env file; *_FILE variants are accepted as well so a
container platform can mount secrets without copying them into the image.
Laboratory 4 adds S3 settings. The API uses a restricted application identity;
only the one-shot bootstrap service receives the administrative identity.
"""
from __future__ import annotations
import os
import socket
from dataclasses import dataclass
from pathlib import Path
def _value(name: str, fallback_name: str | None = None, default: str | None = None) -> str:
raw = os.environ.get(name)
if raw:
return raw
if fallback_name:
filename = os.environ.get(fallback_name)
if filename:
path = Path(filename)
text = path.read_text(encoding="utf-8").strip()
if text:
return text
if default is not None:
return default
raise RuntimeError(f"missing required configuration: {name}")
def _positive_int(name: str, default: int) -> int:
raw = os.environ.get(name, str(default))
try:
value = int(raw)
except ValueError as exc:
raise RuntimeError(f"{name} must be an integer") from exc
if value < 1:
raise RuntimeError(f"{name} must be positive")
return value
def _positive_float(name: str, default: float) -> float:
raw = os.environ.get(name, str(default))
try:
value = float(raw)
except ValueError as exc:
raise RuntimeError(f"{name} must be a number") from exc
if value <= 0:
raise RuntimeError(f"{name} must be positive")
return value
def _instance_id() -> str:
"""Stable public alias of one API process/container.
Containers use their hostname (the short container id) by default. An
explicit INSTANCE_ID is useful for non-container deployments or for a
randomly generated lab7 alias. The value never contains host IP data.
"""
value = (os.environ.get("INSTANCE_ID") or socket.gethostname()).strip()
if not value or len(value) > 128 or any(not (c.isalnum() or c in "-._:") for c in value):
raise RuntimeError("INSTANCE_ID must be 1..128 safe characters")
return value
@dataclass(frozen=True)
class Settings:
database_url: str
cache_url: str
instance_id: str
session_ttl_seconds: int
idempotency_ttl_seconds: int
db_pool_min: int
db_pool_max: int
db_pool_acquire_timeout: float
db_connect_timeout: int
db_statement_timeout_ms: int
redis_connect_timeout: float
redis_socket_timeout: float
max_body_bytes: int
max_upload_bytes: int
max_image_pixels: int
max_decode_concurrency: int
upload_temp_dir: str
s3_endpoint: str
s3_public_endpoint: str
s3_region: str
s3_bucket: str
s3_access_key_id: str
s3_secret_access_key: str
s3_connect_timeout: float
s3_read_timeout: float
presign_ttl_seconds: int
presign_grace_seconds: int
orphan_grace_seconds: int
def load_settings() -> Settings:
return Settings(
database_url=_value("DATABASE_URL", "DATABASE_URL_FILE"),
cache_url=_value("CACHE_URL", "CACHE_URL_FILE"),
instance_id=_instance_id(),
session_ttl_seconds=_positive_int("SESSION_TTL_SECONDS", 3600),
idempotency_ttl_seconds=_positive_int("IDEMPOTENCY_TTL_SECONDS", 86400),
db_pool_min=_positive_int("DB_POOL_MIN", 1),
db_pool_max=_positive_int("DB_POOL_MAX", 10),
db_pool_acquire_timeout=float(os.environ.get("DB_POOL_ACQUIRE_TIMEOUT_SECONDS", "3")),
db_connect_timeout=_positive_int("DB_CONNECT_TIMEOUT_SECONDS", 3),
db_statement_timeout_ms=_positive_int("DB_STATEMENT_TIMEOUT_MS", 5000),
redis_connect_timeout=float(os.environ.get("REDIS_CONNECT_TIMEOUT_SECONDS", "2")),
redis_socket_timeout=float(os.environ.get("REDIS_SOCKET_TIMEOUT_SECONDS", "2")),
max_body_bytes=_positive_int("MAX_BODY_BYTES", 1024 * 1024),
max_upload_bytes=_positive_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024),
max_image_pixels=_positive_int("MAX_IMAGE_PIXELS", 40_000_000),
max_decode_concurrency=_positive_int("MAX_DECODE_CONCURRENCY", 1),
upload_temp_dir=os.environ.get("UPLOAD_TEMP_DIR", "/tmp"),
s3_endpoint=_value("S3_ENDPOINT", "S3_ENDPOINT_FILE", "http://seaweedfs:8333"),
s3_public_endpoint=_value("S3_PUBLIC_ENDPOINT", "S3_PUBLIC_ENDPOINT_FILE", "http://localhost:9000"),
s3_region=_value("S3_REGION", default="us-east-1"),
s3_bucket=_value("S3_BUCKET", default="messenger-attachments"),
s3_access_key_id=_value("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"),
s3_secret_access_key=_value("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"),
s3_connect_timeout=_positive_float("S3_CONNECT_TIMEOUT_SECONDS", 3.0),
s3_read_timeout=_positive_float("S3_READ_TIMEOUT_SECONDS", 30.0),
presign_ttl_seconds=_positive_int("PRESIGN_TTL_SECONDS", 300),
presign_grace_seconds=_positive_int("PRESIGN_GRACE_SECONDS", 3600),
orphan_grace_seconds=_positive_int("ORPHAN_GRACE_SECONDS", 86400),
)
settings = load_settings()
+174
View File
@@ -0,0 +1,174 @@
"""PostgreSQL access: bounded pool, timeouts, and versioned migrations.
Laboratory 3 separates schema management from request serving. The one-shot
``migrate`` Compose service calls :func:`apply_migrations`; API replicas only
open their pool and wait until the newest migration is visible. That prevents
N replicas from racing each other on DDL during a deploy.
"""
from __future__ import annotations
import logging
import threading
import time
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import settings
log = logging.getLogger("messenger.db")
MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "migrations"
LOCK_ID = 0x4D545553 # stable application-wide advisory lock
_pool: ConnectionPool | None = None
_pool_lock = threading.Lock()
_ready = threading.Event()
_started = False
_start_lock = threading.Lock()
def _create_pool() -> ConnectionPool:
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
"-c idle_in_transaction_session_timeout=10000 "
"-c lock_timeout=3000"
)
return ConnectionPool(
conninfo=settings.database_url,
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_pool_acquire_timeout,
open=False,
kwargs={
"row_factory": dict_row,
"connect_timeout": settings.db_connect_timeout,
"application_name": "mtusi-messenger-lab3",
"options": options,
},
)
def pool() -> ConnectionPool:
global _pool
with _pool_lock:
if _pool is None:
_pool = _create_pool()
return _pool
def required_version() -> str:
"""Newest migration shipped in this image; readiness waits for it."""
versions = sorted(path.stem for path in MIGRATIONS_DIR.glob("*.sql"))
if not versions:
raise RuntimeError("no migrations found")
return versions[-1]
def open_pool(timeout: float = 60.0) -> None:
"""Open the pool with bounded retries; used by migrate and API startup."""
deadline = time.monotonic() + timeout
backoff = 0.5
while True:
try:
pool().open(wait=True, timeout=min(5.0, max(0.5, deadline - time.monotonic())))
return
except Exception as exc:
if time.monotonic() >= deadline:
raise
log.warning("database is not reachable yet: %s", type(exc).__name__)
time.sleep(backoff)
backoff = min(backoff * 2, 5.0)
def apply_migrations() -> list[str]:
"""Apply every migration exactly once.
A session-level advisory lock serialises the one-shot migrate service and
any explicit operator run. Each migration and its schema_migrations row
share one transaction, so a failed DDL statement leaves no version row.
"""
applied_now: list[str] = []
with pool().connection() as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
try:
conn.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version text PRIMARY KEY,
applied_at timestamptz NOT NULL DEFAULT clock_timestamp()
)
"""
)
applied = {
row["version"]
for row in conn.execute("SELECT version FROM schema_migrations").fetchall()
}
for path in sorted(MIGRATIONS_DIR.glob("*.sql")):
version = path.stem
if version in applied:
continue
log.info("applying migration %s", version)
conn.execute(path.read_text(encoding="utf-8"))
conn.execute(
"INSERT INTO schema_migrations(version) VALUES (%s)", (version,)
)
applied_now.append(version)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
return applied_now
def schema_is_current() -> bool:
"""Cheap readiness probe: the newest migration is committed and visible."""
if _pool is None:
return False
try:
with pool().connection(timeout=1.0) as conn:
row = conn.execute(
"SELECT 1 FROM schema_migrations WHERE version = %s",
(required_version(),),
).fetchone()
return row is not None
except Exception:
return False
def start_background_initialisation() -> None:
"""Open the pool and wait for the one-shot migrator, without blocking the
liveness endpoint. The API never performs DDL itself."""
global _started
with _start_lock:
if _started:
return
_started = True
def run() -> None:
backoff = 0.5
opened = False
while True:
try:
if not opened:
open_pool(timeout=5.0)
opened = True
if schema_is_current():
_ready.set()
log.info("database pool is ready; schema %s is current", required_version())
return
log.warning("waiting for migrate service: schema %s is not ready", required_version())
except Exception as exc:
log.warning("database initialisation is not ready: %s", type(exc).__name__)
time.sleep(backoff)
backoff = min(backoff * 2, 5.0)
threading.Thread(target=run, name="db-init", daemon=True).start()
def is_ready() -> bool:
return _ready.is_set()
def check_ready() -> bool:
"""Readiness checks the live connection and the migrated schema each time."""
return is_ready() and schema_is_current()
+44
View File
@@ -0,0 +1,44 @@
"""Typed public errors. Internal details never cross the HTTP boundary."""
from __future__ import annotations
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: dict[str, str] | None = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def bad_request(message: str = "Некорректный запрос") -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(*, basic: bool = False) -> ApiError:
challenge = (
'Basic realm="messenger", charset="UTF-8"'
if basic
else 'Bearer realm="messenger"'
)
return ApiError(401, "unauthorized", "Требуется аутентификация", {"WWW-Authenticate": challenge})
def not_found(message: str = "Объект не найден") -> ApiError:
return ApiError(404, "not_found", message)
def conflict(message: str = "Конфликт") -> ApiError:
return ApiError(409, "conflict", message)
def payload_too_large(message: str = "Файл слишком велик") -> ApiError:
return ApiError(413, "payload_too_large", message)
def unsupported_media_type(message: str = "Неподдерживаемый тип файла") -> ApiError:
return ApiError(415, "unsupported_media_type", message)
def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
return ApiError(503, "unavailable", message)
+179
View File
@@ -0,0 +1,179 @@
"""Shared, transactional idempotency for message creation.
The unique scope is (user, method, path, key). The idempotency row, the
message row and its attachment links are written in one PostgreSQL
transaction, so a crash or rollback cannot leave a committed key without its
message. Concurrent retries through different API replicas converge on the
same primary-key row: the loser of the INSERT ... ON CONFLICT waits for the
winner to commit and then reads the original message.
Only the canonical body is hashed, so JSON key order and an omitted
``attachment_ids`` equal to ``[]`` are the same request.
"""
from __future__ import annotations
import hashlib
import json
import secrets
import uuid
from datetime import datetime
from . import db
from .config import settings
from .errors import conflict, not_found
def fingerprint(text: str, attachment_ids: list[str]) -> str:
payload = {
"text": text,
"attachment_ids": sorted(attachment_ids),
}
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
def _cleanup_expired(conn) -> None:
"""Best-effort maintenance; never on the critical path of every request."""
if secrets.randbelow(100) != 0:
return
conn.execute(
"""
DELETE FROM idempotency_keys
WHERE ctid IN (
SELECT ctid FROM idempotency_keys
WHERE expires_at <= clock_timestamp()
LIMIT 1000
)
"""
)
def _validate_attachments(conn, chat_id: uuid.UUID, attachment_ids: list[str]) -> None:
if not attachment_ids:
return
parsed = [uuid.UUID(item) for item in attachment_ids]
rows = conn.execute(
"""
SELECT id, chat_id, status
FROM attachments
WHERE id = ANY(%s)
FOR SHARE
""",
(parsed,),
).fetchall()
if len(rows) != len(parsed):
raise not_found("Вложение не найдено")
by_id = {str(row["id"]): row for row in rows}
for attachment_id in attachment_ids:
row = by_id.get(attachment_id)
if row is None or row["chat_id"] != chat_id:
raise not_found("Вложение не найдено в этом чате")
if row["status"] != "ready":
raise conflict("Вложение ещё не готово")
def _message_with_attachments(conn, message_id) -> dict:
row = conn.execute(
"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
) AS attachment_ids
FROM messages m
WHERE m.id = %s
""",
(message_id,),
).fetchone()
if row is None:
raise conflict("Результат ещё не готов, повторите запрос")
return row
def create_message(
*,
chat_id: uuid.UUID,
actor_id: uuid.UUID,
path: str,
key: str,
text: str,
attachment_ids: list[str],
) -> dict:
"""Return the original or newly created message row, atomically."""
request_hash = fingerprint(text, attachment_ids)
message_id = uuid.uuid4()
with db.pool().connection() as conn:
with conn.transaction():
_validate_attachments(conn, chat_id, attachment_ids)
# An expired key may be reused: remove it inside the same scope.
conn.execute(
"""
DELETE FROM idempotency_keys
WHERE user_id = %s AND method = 'POST' AND path = %s AND key = %s
AND expires_at <= clock_timestamp()
""",
(actor_id, path, key),
)
_cleanup_expired(conn)
inserted = conn.execute(
"""
INSERT INTO idempotency_keys
(user_id, method, path, key, request_hash, message_id, expires_at)
VALUES
(%s, 'POST', %s, %s, %s, %s,
clock_timestamp() + (%s * interval '1 second'))
ON CONFLICT (user_id, method, path, key) DO NOTHING
RETURNING message_id
""",
(
actor_id,
path,
key,
request_hash,
message_id,
settings.idempotency_ttl_seconds,
),
).fetchone()
if inserted is not None:
conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text)
VALUES (%s, %s, %s, %s)
""",
(inserted["message_id"], chat_id, actor_id, text),
)
if attachment_ids:
with conn.cursor() as cursor:
cursor.executemany(
"""
INSERT INTO message_attachments
(message_id, attachment_id, position)
VALUES (%s, %s, %s)
""",
[
(inserted["message_id"], uuid.UUID(item), position)
for position, item in enumerate(attachment_ids)
],
)
return _message_with_attachments(conn, inserted["message_id"])
existing = conn.execute(
"""
SELECT request_hash, message_id, expires_at
FROM idempotency_keys
WHERE user_id = %s AND method = 'POST' AND path = %s AND key = %s
""",
(actor_id, path, key),
).fetchone()
if existing is None:
# The row disappeared between the conflict and this read (only
# possible with a concurrent expiry cleanup). 409 asks the
# client to retry rather than creating an ambiguous duplicate.
raise conflict("Повторите запрос: ключ идемпотентности истёк")
if existing["request_hash"] != request_hash:
raise conflict("Idempotency-Key уже использован с другим сообщением")
return _message_with_attachments(conn, existing["message_id"])
+746
View File
@@ -0,0 +1,746 @@
"""HTTP API for laboratory 3 (PostgreSQL + Redis sessions behind a LB)."""
from __future__ import annotations
import json
import logging
import os
import time
import uuid
from datetime import datetime, timedelta, timezone
from urllib.parse import quote
import psycopg
import psycopg_pool
import redis
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerifyMismatchError
from flask import Flask, Response, g, request
from . import attachments as attachment_store
from . import db, idempotency, s3, sessions
from .config import settings
from .errors import (
ApiError,
bad_request,
conflict,
not_found,
payload_too_large,
unauthorized,
unavailable,
unsupported_media_type,
)
from .validation import (
byte_size,
cursor_time,
decode_cursor,
display_name,
encode_cursor,
exact_keys,
filename,
idempotency_key,
image_content_type,
json_object,
limit_arg,
message_text,
non_blank,
password,
sha256_hex,
username,
uuid_list,
uuid_value,
)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
log = logging.getLogger("messenger.api")
app = Flask(__name__)
# Uploads stream through the handler; the Flask ceiling is deliberately just
# above the contract limit so a single oversized request is rejected early.
app.config["MAX_CONTENT_LENGTH"] = settings.max_upload_bytes + 64 * 1024
hasher = PasswordHasher()
# ---------------------------------------------------------------------------
# Response helpers and middleware
# ---------------------------------------------------------------------------
def _json(payload: object, status: int = 200, headers: dict[str, str] | None = None) -> Response:
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":"))
response = Response(body, status=status, mimetype="application/json")
for key, value in (headers or {}).items():
response.headers[key] = value
return response
def _error_response(error: ApiError, request_id: str) -> Response:
return _json(
{
"error": {
"code": error.code,
"message": error.message,
"request_id": request_id,
}
},
error.status,
error.headers,
)
@app.before_request
def _request_context() -> None:
g.request_id = uuid.uuid4().hex
g.started_at = time.monotonic()
db.start_background_initialisation()
@app.after_request
def _response_headers(response: Response) -> Response:
response.headers["X-Request-Id"] = g.get("request_id", uuid.uuid4().hex)
response.headers["X-Instance-Id"] = settings.instance_id
response.headers["Cache-Control"] = "no-store"
return response
@app.errorhandler(ApiError)
def _handle_api_error(error: ApiError) -> Response:
return _error_response(error, g.get("request_id", uuid.uuid4().hex))
@app.errorhandler(413)
def _handle_too_large(_error: object) -> Response:
return _error_response(
ApiError(413, "payload_too_large", "Тело запроса слишком велико"),
g.get("request_id", uuid.uuid4().hex),
)
@app.errorhandler(404)
def _handle_not_found(_error: object) -> Response:
return _error_response(not_found(), g.get("request_id", uuid.uuid4().hex))
@app.errorhandler(405)
def _handle_method(_error: object) -> Response:
return _error_response(bad_request("Метод не поддерживается"), g.get("request_id", uuid.uuid4().hex))
@app.errorhandler(Exception)
def _handle_unexpected(error: Exception) -> Response:
request_id = g.get("request_id", uuid.uuid4().hex)
if isinstance(error, (psycopg.OperationalError, psycopg_pool.PoolTimeout, psycopg_pool.PoolClosed)):
log.warning("database unavailable on %s %s: %s", request.method, request.path, type(error).__name__)
return _error_response(unavailable("База данных недоступна"), request_id)
if isinstance(error, redis.RedisError):
log.warning("session store unavailable on %s %s: %s", request.method, request.path, type(error).__name__)
return _error_response(unavailable("Хранилище сессий недоступно"), request_id)
if isinstance(error, s3.S3Unavailable):
log.warning("object store unavailable on %s %s", request.method, request.path)
return _error_response(unavailable("Объектное хранилище недоступно"), request_id)
log.exception("unhandled error on %s %s", request.method, request.path)
return _error_response(
ApiError(500, "internal_error", "Внутренняя ошибка сервера"), request_id
)
# ---------------------------------------------------------------------------
# Serialisation
# ---------------------------------------------------------------------------
def format_time(value: datetime) -> str:
return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def user_json(row: dict) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": format_time(row["created_at"]),
}
def chat_json(row: dict) -> dict:
member_ids = sorted(str(item) for item in (row.get("member_ids") or []))
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": format_time(row["created_at"]),
}
def message_json(row: dict) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": format_time(row["created_at"]),
"attachment_ids": [str(item) for item in (row.get("attachment_ids") or [])],
}
# ---------------------------------------------------------------------------
# Authentication
# ---------------------------------------------------------------------------
def _bearer_token() -> str:
header = request.headers.get("Authorization", "")
parts = header.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip():
raise unauthorized()
return parts[1].strip()
def current_user() -> dict:
cached = g.get("current_user")
if cached is not None:
return cached
token = _bearer_token()
user_id, token_digest = sessions.resolve(token)
if not user_id:
raise unauthorized()
try:
parsed = uuid.UUID(user_id)
except ValueError as exc:
raise unauthorized() from exc
with db.pool().connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(parsed,),
).fetchone()
if row is None:
sessions.revoke(token_digest)
raise unauthorized()
g.current_user = row
g.session_token_digest = token_digest
return row
def _basic_credentials() -> tuple[str, str]:
header = request.headers.get("Authorization", "")
parts = header.split(None, 1)
if len(parts) != 2 or parts[0].lower() != "basic":
raise unauthorized(basic=True)
import base64
try:
decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8")
except (ValueError, UnicodeError) as exc:
raise unauthorized(basic=True) from exc
name, separator, secret = decoded.partition(":")
if not separator or not name or not secret:
raise unauthorized(basic=True)
if len(name) > 32 or any(ord(char) > 127 for char in name):
raise unauthorized(basic=True)
return name, secret
def _authenticate_basic() -> dict:
name, secret = _basic_credentials()
with db.pool().connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s",
(name,),
).fetchone()
if row is None:
raise unauthorized(basic=True)
try:
hasher.verify(row["password_hash"], secret)
except (VerifyMismatchError, InvalidHashError):
raise unauthorized(basic=True)
return row
# ---------------------------------------------------------------------------
# Health
# ---------------------------------------------------------------------------
@app.get("/health/live")
def health_live() -> Response:
return _json({"status": "ok"})
@app.get("/health/ready")
def health_ready() -> Response:
if not db.check_ready():
raise unavailable("База данных ещё не готова")
sessions.ping()
if not s3.check_ready():
raise unavailable("Объектное хранилище ещё не готово")
return _json({"status": "ok"})
# ---------------------------------------------------------------------------
# Users and sessions
# ---------------------------------------------------------------------------
@app.post("/api/v1/users")
def register_user() -> Response:
body = json_object(request, settings.max_body_bytes)
exact_keys(body, ("username", "display_name", "password"))
name = username(body["username"])
shown_name = display_name(body["display_name"])
secret = password(body["password"])
user_id = uuid.uuid4()
try:
with db.pool().connection() as conn:
row = conn.execute(
"""
INSERT INTO users (id, username, display_name, password_hash)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(user_id, name, shown_name, hasher.hash(secret)),
).fetchone()
except psycopg.errors.UniqueViolation as exc:
raise conflict("Username уже занят") from exc
return _json(user_json(row), 201)
@app.post("/api/v1/auth/sessions")
def create_session() -> Response:
row = _authenticate_basic()
token, expires_at = sessions.create(str(row["id"]))
return _json(
{
"token": token,
"token_type": "Bearer",
"expires_at": format_time(expires_at),
"user": user_json(row),
},
201,
)
@app.delete("/api/v1/auth/sessions/current")
def revoke_session() -> Response:
current_user() # authentication must happen before revocation
sessions.revoke(g.session_token_digest)
return Response(status=204)
@app.get("/api/v1/users/me")
def current_user_endpoint() -> Response:
return _json(user_json(current_user()))
# ---------------------------------------------------------------------------
# Chats
# ---------------------------------------------------------------------------
def _require_chat_member(chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
with db.pool().connection() as conn:
row = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if row is None:
raise not_found()
@app.post("/api/v1/chats")
def create_chat() -> Response:
actor = current_user()
body = json_object(request, settings.max_body_bytes)
exact_keys(body, ("title", "member_ids"))
title = non_blank(body["title"], "title", 1, 100)
member_ids = uuid_list(body["member_ids"], "member_ids")
actor_id = str(actor["id"])
if actor_id not in member_ids:
raise bad_request("Создатель должен входить в member_ids")
member_uuids = [uuid.UUID(item) for item in member_ids]
chat_id = uuid.uuid4()
with db.pool().connection() as conn:
existing = conn.execute(
"SELECT id FROM users WHERE id = ANY(%s)", (member_uuids,)
).fetchall()
if len(existing) != len(member_uuids):
raise bad_request("Указан неизвестный участник")
row = conn.execute(
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
(chat_id, title),
).fetchone()
with conn.cursor() as cursor:
cursor.executemany(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
[(chat_id, member) for member in member_uuids],
)
payload = dict(row)
payload["member_ids"] = member_uuids
return _json(chat_json(payload), 201)
@app.get("/api/v1/chats")
def list_chats() -> Response:
actor = current_user()
limit = limit_arg(request.args.get("limit"))
cursor = decode_cursor(request.args.get("cursor"))
parameters: list[object] = [actor["id"]]
predicate = ""
if cursor is not None:
if cursor.get("kind") != "chats" or cursor.get("user_id") != str(actor["id"]):
raise bad_request("Некорректный cursor")
try:
cursor_created = cursor_time(cursor.get("created_at"))
cursor_id = uuid_value(cursor.get("id"), "cursor")
except ApiError:
raise bad_request("Некорректный cursor")
predicate = "AND (c.created_at, c.id) > (%s, %s)"
parameters.extend([cursor_created, cursor_id])
parameters.append(limit + 1)
query = f"""
SELECT c.id, c.title, c.created_at,
ARRAY(
SELECT cm.user_id FROM chat_members cm
WHERE cm.chat_id = c.id ORDER BY cm.user_id
) AS member_ids
FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
WHERE TRUE {predicate}
ORDER BY c.created_at, c.id
LIMIT %s
"""
with db.pool().connection() as conn:
rows = conn.execute(query, tuple(parameters)).fetchall()
page = rows[:limit]
next_cursor = None
if len(rows) > limit and page:
last = page[-1]
next_cursor = encode_cursor(
{
"v": 1,
"kind": "chats",
"user_id": str(actor["id"]),
"created_at": format_time(last["created_at"]),
"id": str(last["id"]),
}
)
return _json({"items": [chat_json(row) for row in page], "next_cursor": next_cursor})
@app.get("/api/v1/chats/<chat_id>")
def get_chat(chat_id: str) -> Response:
actor = current_user()
parsed_chat_id = uuid_value(chat_id, "chat_id")
with db.pool().connection() as conn:
row = conn.execute(
"""
SELECT c.id, c.title, c.created_at,
ARRAY(
SELECT cm.user_id FROM chat_members cm
WHERE cm.chat_id = c.id ORDER BY cm.user_id
) AS member_ids
FROM chats c
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
WHERE c.id = %s
""",
(actor["id"], parsed_chat_id),
).fetchone()
if row is None:
raise not_found()
return _json(chat_json(row))
# ---------------------------------------------------------------------------
# Messages
# ---------------------------------------------------------------------------
@app.post("/api/v1/chats/<chat_id>/messages")
def create_message(chat_id: str) -> Response:
actor = current_user()
parsed_chat_id = uuid_value(chat_id, "chat_id")
_require_chat_member(parsed_chat_id, actor["id"])
body = json_object(request, settings.max_body_bytes)
exact_keys(body, ("text",), ("attachment_ids",))
attachments = uuid_list(body.get("attachment_ids", []), "attachment_ids", 0, 10)
text = message_text(body["text"], allow_empty=bool(attachments))
key = idempotency_key(request.headers.get("Idempotency-Key"))
row = idempotency.create_message(
chat_id=parsed_chat_id,
actor_id=actor["id"],
path=request.path,
key=key,
text=text,
attachment_ids=attachments,
)
return _json(message_json(row), 201)
@app.get("/api/v1/chats/<chat_id>/messages")
def list_messages(chat_id: str) -> Response:
actor = current_user()
parsed_chat_id = uuid_value(chat_id, "chat_id")
_require_chat_member(parsed_chat_id, actor["id"])
limit = limit_arg(request.args.get("limit"))
cursor = decode_cursor(request.args.get("cursor"))
parameters: list[object] = [parsed_chat_id]
predicate = ""
if cursor is not None:
if cursor.get("kind") != "messages" or cursor.get("chat_id") != str(parsed_chat_id):
raise bad_request("Некорректный cursor")
try:
cursor_created = cursor_time(cursor.get("created_at"))
cursor_id = uuid_value(cursor.get("id"), "cursor")
except ApiError:
raise bad_request("Некорректный cursor")
predicate = "AND (created_at, id) > (%s, %s)"
parameters.extend([cursor_created, cursor_id])
parameters.append(limit + 1)
query = f"""
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(
SELECT ma.attachment_id
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position
) AS attachment_ids
FROM messages m
WHERE m.chat_id = %s {predicate}
ORDER BY m.created_at, m.id
LIMIT %s
"""
with db.pool().connection() as conn:
rows = conn.execute(query, tuple(parameters)).fetchall()
page = rows[:limit]
next_cursor = None
if len(rows) > limit and page:
last = page[-1]
next_cursor = encode_cursor(
{
"v": 1,
"kind": "messages",
"chat_id": str(parsed_chat_id),
"created_at": format_time(last["created_at"]),
"id": str(last["id"]),
}
)
return _json({"items": [message_json(row) for row in page], "next_cursor": next_cursor})
# ---------------------------------------------------------------------------
# Attachments (lab4)
# ---------------------------------------------------------------------------
def _attachment_for_member(attachment_id: uuid.UUID, actor_id: uuid.UUID) -> dict:
with db.pool().connection() as conn:
row = conn.execute(
"""
SELECT a.*
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id AND cm.user_id = %s
WHERE a.id = %s
""",
(actor_id, attachment_id),
).fetchone()
if row is None:
raise not_found()
return row
def _attachment_owned(attachment_id: uuid.UUID, actor_id: uuid.UUID) -> dict | None:
with db.pool().connection() as conn:
return conn.execute(
"""
SELECT a.*
FROM attachments a
JOIN chat_members cm
ON cm.chat_id = a.chat_id AND cm.user_id = %s
WHERE a.id = %s AND a.owner_id = %s
""",
(actor_id, attachment_id, actor_id),
).fetchone()
@app.post("/api/v1/chats/<chat_id>/attachments")
def upload_attachment(chat_id: str) -> Response:
"""Synchronous proxy upload: 201 only after S3 PUT and PostgreSQL COMMIT."""
actor = current_user()
parsed_chat_id = uuid_value(chat_id, "chat_id")
_require_chat_member(parsed_chat_id, actor["id"])
name = filename(request.args.get("filename"))
declared = image_content_type(request.mimetype)
if request.content_length is not None and request.content_length > settings.max_upload_bytes:
raise payload_too_large("Файл больше 10 MiB")
path = None
try:
path, size, digest = attachment_store.read_to_temp(request.stream)
attachment_store.validate_image(path, declared)
attachment_id = uuid.uuid4()
key = attachment_store.object_key(parsed_chat_id, attachment_id)
with open(path, "rb") as body:
s3.put_file(key, body, declared, size, digest)
try:
with db.pool().connection() as conn:
row = conn.execute(
"""
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, object_key, status)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'ready')
RETURNING *
""",
(
attachment_id,
parsed_chat_id,
actor["id"],
name,
declared,
size,
digest,
key,
),
).fetchone()
except Exception as error:
# Compensate: an S3 object without metadata must not survive.
s3.delete(key)
if isinstance(error, psycopg.Error):
raise unavailable("Не удалось сохранить метаданные вложения") from error
raise
return _json(attachment_store.attachment_json(row), 201)
finally:
attachment_store.cleanup(path)
@app.post("/api/v1/chats/<chat_id>/attachments/presign")
def presign_attachment(chat_id: str) -> Response:
"""Extension: short-lived direct upload URL; the proxy endpoint stays mandatory."""
actor = current_user()
parsed_chat_id = uuid_value(chat_id, "chat_id")
_require_chat_member(parsed_chat_id, actor["id"])
body = json_object(request, settings.max_body_bytes)
exact_keys(body, ("filename", "content_type", "size_bytes", "sha256"))
name = filename(body["filename"])
declared = image_content_type(body["content_type"])
size = byte_size(body["size_bytes"], settings.max_upload_bytes)
digest = sha256_hex(body["sha256"])
attachment_id = uuid.uuid4()
key = attachment_store.object_key(parsed_chat_id, attachment_id)
upload_url = s3.presign_put(key, declared, digest, settings.presign_ttl_seconds)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.presign_ttl_seconds)
try:
with db.pool().connection() as conn:
conn.execute(
"""
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type,
size_bytes, sha256, object_key, status, presign_expires_at)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'pending', %s)
""",
(
attachment_id,
parsed_chat_id,
actor["id"],
name,
declared,
size,
digest,
key,
expires_at,
),
)
except psycopg.Error as error:
raise unavailable("Не удалось сохранить метаданные вложения") from error
return _json(
{
"attachment_id": str(attachment_id),
"upload_url": upload_url,
"method": "PUT",
"headers": {"Content-Type": declared, "x-amz-meta-sha256": digest},
"expires_at": format_time(expires_at),
},
201,
)
@app.post("/api/v1/attachments/<attachment_id>/complete")
def complete_attachment(attachment_id: str) -> Response:
"""Extension: verify a direct upload and make the attachment ready."""
actor = current_user()
parsed_id = uuid_value(attachment_id, "attachment_id")
row = _attachment_owned(parsed_id, actor["id"])
if row is None:
raise not_found()
body = json_object(request, settings.max_body_bytes)
exact_keys(body, ("sha256", "size_bytes"))
digest = sha256_hex(body["sha256"])
size = byte_size(body["size_bytes"], settings.max_upload_bytes)
if row["status"] == "ready":
if row["sha256"] == digest and int(row["size_bytes"]) == size:
return _json(attachment_store.attachment_json(row))
raise conflict("Вложение уже завершено с другими параметрами")
if row["status"] != "pending":
raise conflict("Вложение не может быть завершено")
expires_at = row.get("presign_expires_at")
if expires_at is not None and expires_at <= datetime.now(timezone.utc):
raise conflict("Срок действия загрузки истёк")
head = s3.head(row["object_key"])
if head is None:
raise conflict("Объект ещё не загружен")
if int(head.get("ContentLength", -1)) != size:
raise bad_request("Размер загруженного объекта не совпадает")
actual_size, actual_digest = s3.stream_sha256(row["object_key"])
if actual_size != size or actual_digest != digest:
# Wrong bytes are never promoted; delete them so a retry starts clean.
s3.delete(row["object_key"])
raise bad_request("Контрольная сумма загруженного объекта не совпадает")
with db.pool().connection() as conn:
updated = conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, updated_at = clock_timestamp()
WHERE id = %s AND status = 'pending'
RETURNING *
""",
(parsed_id,),
).fetchone()
if updated is None:
raise conflict("Вложение уже обработано")
return _json(attachment_store.attachment_json(updated))
@app.get("/api/v1/attachments/<attachment_id>")
def get_attachment(attachment_id: str) -> Response:
actor = current_user()
parsed_id = uuid_value(attachment_id, "attachment_id")
row = _attachment_for_member(parsed_id, actor["id"])
return _json(attachment_store.attachment_json(row))
@app.get("/api/v1/attachments/<attachment_id>/content")
def download_attachment(attachment_id: str) -> Response:
actor = current_user()
parsed_id = uuid_value(attachment_id, "attachment_id")
variant = request.args.get("variant", "original")
if variant != "original":
raise bad_request("variant должен быть original")
row = _attachment_for_member(parsed_id, actor["id"])
if row["status"] != "ready":
raise conflict("Вложение ещё не готово")
try:
body = s3.open_stream(row["object_key"])
except s3.S3Missing as error:
raise unavailable("Оригинал вложения недоступен") from error
def chunks():
try:
for chunk in body.iter_chunks(chunk_size=64 * 1024):
yield chunk
finally:
body.close()
response = Response(chunks(), mimetype=row["content_type"])
response.headers["Content-Length"] = str(row["size_bytes"])
response.headers["Content-Disposition"] = (
"inline; filename*=UTF-8''" + quote(row["filename"], safe="")
)
return response
+37
View File
@@ -0,0 +1,37 @@
"""One-shot schema migrator.
Run as ``python -m app.migrate`` from the dedicated Compose service. It waits
for PostgreSQL, takes the application advisory lock and applies only missing
migration files. Exiting non-zero blocks API replicas via
``depends_on: service_completed_successfully``.
"""
from __future__ import annotations
import logging
import sys
from . import db
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
log = logging.getLogger("messenger.migrate")
def main() -> int:
log.info("waiting for PostgreSQL and applying migrations")
db.open_pool(timeout=60.0)
applied = db.apply_migrations()
if applied:
log.info("applied migrations: %s", ", ".join(applied))
else:
log.info("schema already current at %s", db.required_version())
with db.pool().connection() as conn:
rows = conn.execute(
"SELECT version, applied_at FROM schema_migrations ORDER BY version"
).fetchall()
for row in rows:
log.info("schema version %s applied at %s", row["version"], row["applied_at"].isoformat())
return 0
if __name__ == "__main__":
sys.exit(main())
+136
View File
@@ -0,0 +1,136 @@
"""Safe, repeatable S3/PostgreSQL reconciler (lab4 operational backstop).
What it does
------------
* finds S3 objects with no attachment row ("orphans") and deletes only those
older than a grace period;
* finds pending presigned uploads older than a grace period and removes both
the row and any object, so an interrupted direct upload cannot leak forever;
* reports ready rows whose object is missing and records ``missing_since`` for
operators without deleting user-visible metadata.
``--dry-run`` is the default. ``--apply`` performs the deletions. Re-running
after an interruption is safe: every action is idempotent.
"""
from __future__ import annotations
import argparse
import json
import logging
import sys
from datetime import datetime, timezone
from . import db, s3
from .config import settings
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
log = logging.getLogger("messenger.reconcile")
def _age_seconds(now: datetime, value: datetime | None) -> float | None:
if value is None:
return None
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return (now - value.astimezone(timezone.utc)).total_seconds()
def run(*, apply: bool, orphan_grace: int, pending_grace: int, prefix: str) -> dict:
now = datetime.now(timezone.utc)
db.open_pool(timeout=60.0)
objects = s3.list_objects(prefix)
object_keys = {item["key"] for item in objects}
with db.pool().connection() as conn:
rows = conn.execute(
"SELECT id, object_key, status, created_at, missing_since FROM attachments"
).fetchall()
db_keys = {row["object_key"] for row in rows}
orphans = [item for item in objects if item["key"] not in db_keys]
stale_orphans = []
fresh_orphans = []
for item in orphans:
age = _age_seconds(now, item["last_modified"])
if age is None or age >= orphan_grace:
stale_orphans.append(item)
else:
fresh_orphans.append(item)
stale_pending = [
row
for row in rows
if row["status"] == "pending"
and (_age_seconds(now, row["created_at"]) or 0) >= pending_grace
]
missing = [
row
for row in rows
if row["status"] == "ready" and row["object_key"] not in object_keys
]
deleted_orphans = 0
deleted_pending = 0
marked_missing = 0
if apply:
for item in stale_orphans:
s3.delete(item["key"])
deleted_orphans += 1
for row in stale_pending:
s3.delete(row["object_key"])
with db.pool().connection() as conn:
conn.execute(
"DELETE FROM attachments WHERE id = %s AND status = 'pending'",
(row["id"],),
)
deleted_pending += 1
for row in missing:
if row["missing_since"] is None:
with db.pool().connection() as conn:
conn.execute(
"UPDATE attachments SET missing_since = clock_timestamp() WHERE id = %s",
(row["id"],),
)
marked_missing += 1
return {
"apply": apply,
"grace_seconds": {"orphan": orphan_grace, "pending": pending_grace},
"objects": len(objects),
"metadata_rows": len(rows),
"orphans_total": len(orphans),
"orphans_fresh_kept": len(fresh_orphans),
"orphans_stale_deleted": deleted_orphans if apply else len(stale_orphans),
"stale_pending_deleted": deleted_pending if apply else len(stale_pending),
"missing_objects_reported": len(missing),
"missing_objects_marked": marked_missing,
"sample_orphan_keys": [item["key"] for item in stale_orphans[:5]],
"sample_missing_keys": [row["object_key"] for row in missing[:5]],
}
def main() -> int:
parser = argparse.ArgumentParser(description="Reconcile private S3 objects with attachment metadata")
parser.add_argument("--apply", action="store_true", help="delete stale orphans/pending rows (default: dry-run)")
parser.add_argument("--dry-run", action="store_true", help="explicit dry-run (default)")
parser.add_argument("--orphan-grace-seconds", type=int, default=settings.orphan_grace_seconds)
parser.add_argument("--pending-grace-seconds", type=int, default=settings.presign_grace_seconds)
parser.add_argument("--prefix", default="")
args = parser.parse_args()
apply = args.apply and not args.dry_run
try:
summary = run(
apply=apply,
orphan_grace=max(0, args.orphan_grace_seconds),
pending_grace=max(0, args.pending_grace_seconds),
prefix=args.prefix,
)
except s3.S3Unavailable as error:
log.error("object store is unavailable: %s", type(error).__name__)
return 2
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
return 0
if __name__ == "__main__":
sys.exit(main())
+230
View File
@@ -0,0 +1,230 @@
"""Thin, bounded S3 client wrapper for the attachment store.
The API identity has only Read/Write/List on the private bucket. The
administrative identity is used exclusively by ``app.s3_bootstrap``.
All boto3 calls are wrapped so that HTTP handlers never leak stack traces or
credentials. Large bodies are streamed; no method here reads a whole object
into memory.
"""
from __future__ import annotations
import hashlib
import threading
from datetime import datetime, timezone
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .config import settings
class S3Unavailable(Exception):
"""The object store is unreachable or rejected a request."""
class S3Missing(Exception):
"""The requested object does not exist."""
_clients: dict[bool, object] = {}
_clients_lock = threading.Lock()
def _make_client(admin: bool = False):
if admin:
access_key = _required_admin("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE")
secret_key = _required_admin("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE")
else:
access_key = settings.s3_access_key_id
secret_key = settings.s3_secret_access_key
config = Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=settings.s3_connect_timeout,
read_timeout=settings.s3_read_timeout,
retries={"max_attempts": 2, "mode": "standard"},
max_pool_connections=32,
)
return boto3.client(
"s3",
endpoint_url=settings.s3_endpoint,
region_name=settings.s3_region,
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=config,
)
def _required_admin(name: str, fallback: str) -> str:
import os
from pathlib import Path
raw = os.environ.get(name)
if raw:
return raw
filename = os.environ.get(fallback)
if filename:
return Path(filename).read_text(encoding="utf-8").strip()
raise RuntimeError(f"missing required configuration: {name}")
def client(admin: bool = False):
with _clients_lock:
cached = _clients.get(admin)
if cached is None:
cached = _make_client(admin)
_clients[admin] = cached
return cached
def _error_code(error: ClientError) -> str:
return str(error.response.get("Error", {}).get("Code", ""))
def _missing(error: ClientError) -> bool:
return _error_code(error) in ("404", "NoSuchKey", "NoSuchBucket", "NotFound")
def ensure_bucket() -> None:
"""Idempotent private-bucket bootstrap using the administrative identity."""
s3 = client(admin=True)
try:
s3.head_bucket(Bucket=settings.s3_bucket)
return
except ClientError as error:
if _error_code(error) not in ("404", "NoSuchBucket", "NotFound"):
raise S3Unavailable("S3 head_bucket failed") from error
try:
s3.create_bucket(Bucket=settings.s3_bucket)
except ClientError as error:
code = _error_code(error)
if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
raise S3Unavailable("S3 create_bucket failed") from error
try:
s3.head_bucket(Bucket=settings.s3_bucket)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("bucket is not usable after bootstrap") from error
def check_ready() -> bool:
try:
client().head_bucket(Bucket=settings.s3_bucket)
return True
except Exception:
return False
def put_file(key: str, fileobj, content_type: str, size: int, sha256: str) -> None:
try:
client().put_object(
Bucket=settings.s3_bucket,
Key=key,
Body=fileobj,
ContentType=content_type,
ContentLength=size,
Metadata={"sha256": sha256},
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("S3 upload failed") from error
def delete(key: str) -> None:
try:
client().delete_object(Bucket=settings.s3_bucket, Key=key)
except (BotoCoreError, ClientError):
# Compensation is best effort; the reconciler is the backstop.
return
def head(key: str) -> dict | None:
try:
return client().head_object(Bucket=settings.s3_bucket, Key=key)
except ClientError as error:
if _missing(error):
return None
raise S3Unavailable("S3 head_object failed") from error
except BotoCoreError as error:
raise S3Unavailable("S3 head_object failed") from error
def open_stream(key: str):
"""Return a streaming body. The caller must close it."""
try:
return client().get_object(Bucket=settings.s3_bucket, Key=key)["Body"]
except ClientError as error:
if _missing(error):
raise S3Missing("object is missing") from error
raise S3Unavailable("S3 get_object failed") from error
except BotoCoreError as error:
raise S3Unavailable("S3 get_object failed") from error
def stream_sha256(key: str) -> tuple[int, str]:
"""Read an object once, bounded by 64 KiB chunks, and return size+digest."""
body = open_stream(key)
digest = hashlib.sha256()
total = 0
try:
for chunk in body.iter_chunks(chunk_size=64 * 1024):
total += len(chunk)
digest.update(chunk)
except (BotoCoreError, ClientError, OSError) as error:
raise S3Unavailable("S3 streaming read failed") from error
finally:
body.close()
return total, digest.hexdigest()
def presign_put(key: str, content_type: str, sha256: str, expires: int) -> str:
"""Create a short-lived presigned PUT URL on the public S3 endpoint.
The API's internal endpoint is not reachable from user clients, so the
presigner uses S3_PUBLIC_ENDPOINT. The bucket stays private: the URL is
the only grant and it expires quickly.
"""
presigner = boto3.client(
"s3",
endpoint_url=settings.s3_public_endpoint,
region_name=settings.s3_region,
aws_access_key_id=settings.s3_access_key_id,
aws_secret_access_key=settings.s3_secret_access_key,
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
try:
return presigner.generate_presigned_url(
"put_object",
Params={
"Bucket": settings.s3_bucket,
"Key": key,
"ContentType": content_type,
"Metadata": {"sha256": sha256},
},
ExpiresIn=expires,
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("could not sign upload URL") from error
def list_objects(prefix: str = "") -> list[dict]:
"""List bucket objects with key, size and LastModified (UTC)."""
s3 = client()
result: list[dict] = []
try:
paginator = s3.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix):
for item in page.get("Contents", []):
modified = item.get("LastModified")
if isinstance(modified, datetime) and modified.tzinfo is None:
modified = modified.replace(tzinfo=timezone.utc)
result.append(
{
"key": item["Key"],
"size": int(item.get("Size", 0)),
"last_modified": modified,
}
)
except (BotoCoreError, ClientError) as error:
raise S3Unavailable("S3 list_objects failed") from error
return result
+39
View File
@@ -0,0 +1,39 @@
"""One-shot, idempotent private-bucket bootstrap.
Runs in the dedicated ``s3-init`` Compose service with the administrative S3
identity. The API replicas start only after this service exits successfully,
and they use a different, restricted identity.
"""
from __future__ import annotations
import logging
import sys
import time
from . import s3
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
log = logging.getLogger("messenger.s3-bootstrap")
def main() -> int:
deadline = time.monotonic() + 90.0
backoff = 0.5
while True:
try:
s3.ensure_bucket()
if not s3.check_ready():
raise RuntimeError("restricted identity cannot see the bucket")
log.info("private bucket is present and visible to the API identity")
return 0
except Exception as error:
if time.monotonic() >= deadline:
log.error("bucket bootstrap failed: %s", type(error).__name__)
return 1
log.warning("bucket bootstrap is not ready yet: %s", type(error).__name__)
time.sleep(backoff)
backoff = min(backoff * 2, 5.0)
if __name__ == "__main__":
sys.exit(main())
+101
View File
@@ -0,0 +1,101 @@
"""Opaque bearer sessions stored in Redis/Valkey with a TTL.
PostgreSQL is the source of truth for users/chats/messages. Redis is the
authoritative store for *active sessions*; its append-only file and named
volume make those sessions survive a restart. The raw bearer token is never
stored: only a SHA-256 digest is used as the Redis key.
"""
from __future__ import annotations
import hashlib
import json
import secrets
from datetime import datetime, timedelta, timezone
import redis
from .config import settings
from .errors import unavailable
_client: redis.Redis | None = None
def client() -> redis.Redis:
global _client
if _client is None:
_client = redis.Redis.from_url(
settings.cache_url,
decode_responses=True,
socket_connect_timeout=settings.redis_connect_timeout,
socket_timeout=settings.redis_socket_timeout,
retry_on_timeout=False,
health_check_interval=30,
)
return _client
def digest(token: str) -> str:
return hashlib.sha256(token.encode("utf-8")).hexdigest()
def _key(token_digest: str) -> str:
return "session:" + token_digest
def create(user_id: str) -> tuple[str, datetime]:
token = secrets.token_urlsafe(32) # 256 bits of entropy, opaque to clients
ttl = settings.session_ttl_seconds
expires_at = datetime.now(timezone.utc) + timedelta(seconds=ttl)
payload = json.dumps(
{"user_id": user_id, "expires_at": expires_at.isoformat().replace("+00:00", "Z")},
separators=(",", ":"),
)
try:
client().setex(_key(digest(token)), ttl, payload)
except redis.RedisError as exc:
raise unavailable("Хранилище сессий недоступно") from exc
return token, expires_at
def resolve(token: str) -> tuple[str, str]:
"""Return (user_id, token_digest) or raise a bounded 503 error."""
token_digest = digest(token)
try:
raw = client().get(_key(token_digest))
except redis.RedisError as exc:
raise unavailable("Хранилище сессий недоступно") from exc
if not raw:
return "", token_digest
try:
data = json.loads(raw)
user_id = str(data["user_id"])
expires_at = datetime.fromisoformat(str(data["expires_at"]).replace("Z", "+00:00"))
except (ValueError, KeyError, TypeError):
return "", token_digest
if expires_at <= datetime.now(timezone.utc):
revoke(token_digest)
return "", token_digest
return user_id, token_digest
def revoke(token_digest: str) -> None:
try:
client().delete(_key(token_digest))
except redis.RedisError as exc:
raise unavailable("Хранилище сессий недоступно") from exc
def ping() -> None:
try:
if not client().ping():
raise unavailable("Хранилище сессий недоступно")
except redis.RedisError as exc:
raise unavailable("Хранилище сессий недоступно") from exc
def seconds_left(token_digest: str) -> int:
try:
value = client().ttl(_key(token_digest))
except redis.RedisError:
return 0
return max(0, int(value))
+194
View File
@@ -0,0 +1,194 @@
"""Strict parsing shared by HTTP handlers.
The contract requires rejection of extra JSON fields, wrong types and
whitespace-only required strings. Validation happens before any SQL.
"""
from __future__ import annotations
import base64
import binascii
import json
import re
import uuid
from datetime import datetime, timezone
from flask import Request
from .errors import ApiError, bad_request
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
INTEGER_RE = re.compile(r"^[0-9]+$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
SHA256_RE = re.compile(r"^[a-f0-9]{64}$")
def json_object(request: Request, max_bytes: int) -> dict:
if request.mimetype != "application/json":
raise bad_request("Ожидается application/json")
raw = request.get_data(cache=False)
if not raw:
raise bad_request("Тело запроса обязательно")
if len(raw) > max_bytes:
raise ApiError(413, "payload_too_large", "Тело запроса слишком велико")
try:
value = json.loads(raw)
except (ValueError, UnicodeError) as exc:
raise bad_request("Некорректный JSON") from exc
if not isinstance(value, dict):
raise bad_request("Ожидается JSON-объект")
return value
def exact_keys(value: dict, required: tuple[str, ...], optional: tuple[str, ...] = ()) -> None:
allowed = set(required) | set(optional)
if not set(required).issubset(value) or set(value) - allowed:
raise bad_request("Неверный набор полей")
def text_field(value: object, name: str, minimum: int, maximum: int) -> str:
if not isinstance(value, str):
raise bad_request(f"Поле {name} должно быть строкой")
if not (minimum <= len(value) <= maximum):
raise bad_request(f"Поле {name} имеет недопустимую длину")
return value
def non_blank(value: object, name: str, minimum: int, maximum: int) -> str:
result = text_field(value, name, minimum, maximum)
if not result.strip():
raise bad_request(f"Поле {name} не должно быть пустым")
return result
def username(value: object) -> str:
result = text_field(value, "username", 3, 32)
if USERNAME_RE.fullmatch(result) is None:
raise bad_request("Недопустимый username")
return result
def display_name(value: object) -> str:
return non_blank(value, "display_name", 1, 100)
def password(value: object) -> str:
result = text_field(value, "password", 12, 128)
return result
def message_text(value: object, allow_empty: bool = False) -> str:
if not isinstance(value, str):
raise bad_request("Поле text должно быть строкой")
if len(value) > 4000:
raise bad_request("Поле text имеет недопустимую длину")
if not allow_empty and not value.strip():
raise bad_request("Поле text не должно быть пустым")
return value
def filename(value: object) -> str:
"""Display-only filename; it never becomes an object key.
The contract forbids path separators, NUL and control characters. The
limit is measured in Unicode code points (Python's len on str).
"""
if not isinstance(value, str):
raise bad_request("Параметр filename обязателен")
if not (1 <= len(value) <= 255):
raise bad_request("filename имеет недопустимую длину")
if value in (".", ".."):
raise bad_request("filename недопустим")
for char in value:
code = ord(char)
if char in "/\\" or code == 0 or code < 0x20 or code == 0x7F:
raise bad_request("filename содержит недопустимые символы")
return value
def image_content_type(value: object) -> str:
if not isinstance(value, str) or value not in ("image/png", "image/jpeg"):
raise ApiError(415, "unsupported_media_type", "Ожидается PNG или JPEG")
return value
def sha256_hex(value: object) -> str:
if not isinstance(value, str) or SHA256_RE.fullmatch(value) is None:
raise bad_request("sha256 должен быть lowercase hex SHA-256")
return value
def byte_size(value: object, maximum: int) -> int:
if not isinstance(value, int) or isinstance(value, bool) or not (1 <= value <= maximum):
raise bad_request("size_bytes вне допустимого диапазона")
return value
def idempotency_key(value: object) -> str:
if not isinstance(value, str) or IDEMPOTENCY_KEY_RE.fullmatch(value) is None:
raise bad_request("Некорректный Idempotency-Key")
return value
def uuid_list(value: object, name: str, minimum: int = 1, maximum: int = 100) -> list[str]:
if not isinstance(value, list):
raise bad_request(f"Поле {name} должно быть массивом")
if not (minimum <= len(value) <= maximum):
raise bad_request(f"Поле {name} имеет недопустимую длину")
parsed: list[str] = []
for item in value:
parsed.append(str(uuid_value(item, name)))
if len(set(parsed)) != len(parsed):
raise bad_request(f"Поле {name} содержит дубликаты")
return parsed
def uuid_value(value: object, name: str = "id") -> uuid.UUID:
if not isinstance(value, str):
raise bad_request(f"Поле {name} должно быть UUID")
try:
return uuid.UUID(value)
except (ValueError, AttributeError) as exc:
raise bad_request(f"Поле {name} должно быть UUID") from exc
def limit_arg(raw: str | None, default: int = 50) -> int:
if raw is None:
return default
if INTEGER_RE.fullmatch(raw) is None:
raise bad_request("limit должен быть целым числом")
value = int(raw)
if not (1 <= value <= 100):
raise bad_request("limit должен быть в диапазоне 1..100")
return value
def encode_cursor(payload: dict) -> str:
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(raw: str | None) -> dict | None:
if raw is None:
return None
if not raw or len(raw) > 4096:
raise bad_request("Некорректный cursor")
padded = raw + "=" * (-len(raw) % 4)
try:
data = json.loads(base64.urlsafe_b64decode(padded.encode("ascii")))
except (ValueError, UnicodeError, AttributeError, binascii.Error) as exc:
raise bad_request("Некорректный cursor") from exc
if not isinstance(data, dict):
raise bad_request("Некорректный cursor")
return data
def cursor_time(value: object) -> datetime:
if not isinstance(value, str):
raise bad_request("Некорректный cursor")
try:
result = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError as exc:
raise bad_request("Некорректный cursor") from exc
if result.tzinfo is None:
raise bad_request("Некорректный cursor")
return result.astimezone(timezone.utc)
+254
View File
@@ -0,0 +1,254 @@
# Laboratory 4 stand:
# client -> nginx LB -> two or more API replicas -> PostgreSQL (metadata)
# -> private SeaweedFS S3 (bytes)
#
# Copy .env.example to .env and replace every change-me value before use.
# Only the LB and the S3 API are published; PostgreSQL, Redis, the SeaweedFS
# master/filer UIs and the API replicas stay on the internal network.
#
# The S3 port is published because the optional direct-upload extension gives
# clients a short-lived presigned PUT URL. The bucket itself is private:
# unsigned GET/PUT is rejected by SeaweedFS.
name: mtusi-lab4
x-api-build: &api-build
image: mtusi-lab4-api:local
build:
context: .
dockerfile: Dockerfile
x-api-environment: &api-environment
APP_ENV: ${APP_ENV:-lab}
DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env}
CACHE_URL: ${CACHE_URL:-redis://redis:6379/0}
INSTANCE_ID: ${INSTANCE_ID:-}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_POOL_ACQUIRE_TIMEOUT_SECONDS: ${DB_POOL_ACQUIRE_TIMEOUT_SECONDS:-3}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:-2}
REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:-2}
MAX_BODY_BYTES: ${MAX_BODY_BYTES:-1048576}
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
MAX_DECODE_CONCURRENCY: ${MAX_DECODE_CONCURRENCY:-1}
UPLOAD_TEMP_DIR: /tmp
S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333}
S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://localhost:9000}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-3}
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-30}
PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-300}
PRESIGN_GRACE_SECONDS: ${PRESIGN_GRACE_SECONDS:-3600}
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-86400}
services:
postgres:
image: postgres:16.4-alpine
restart: unless-stopped
environment:
POSTGRES_DB: ${POSTGRES_DB:-messenger}
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env}
command:
- postgres
- -c
- max_connections=120
- -c
- shared_buffers=128MB
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
interval: 3s
timeout: 3s
retries: 20
deploy:
resources:
limits:
cpus: "${POSTGRES_CPU_LIMIT:-1.0}"
memory: "${POSTGRES_MEM_LIMIT:-512M}"
networks:
- backend
redis:
image: redis:7.4.0-alpine
restart: unless-stopped
command:
- redis-server
- --appendonly
- "yes"
- --appendfsync
- everysec
- --dir
- /data
volumes:
- redisdata:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 3s
timeout: 3s
retries: 20
deploy:
resources:
limits:
cpus: "${REDIS_CPU_LIMIT:-0.5}"
memory: "${REDIS_MEM_LIMIT:-256M}"
networks:
- backend
seaweedfs:
image: chrislusf/seaweedfs:3.80
restart: unless-stopped
entrypoint: ["/bin/sh", "/etc/seaweedfs/entrypoint.sh"]
environment:
S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?set S3_ADMIN_ACCESS_KEY_ID in .env}
S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?set S3_ADMIN_SECRET_ACCESS_KEY in .env}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
volumes:
- seaweeddata:/data
- ./infra/seaweedfs/entrypoint.sh:/etc/seaweedfs/entrypoint.sh:ro
ports:
- "${S3_PORT:-9000}:8333"
healthcheck:
test: ["CMD-SHELL", "nc -z 127.0.0.1 8333"]
interval: 3s
timeout: 3s
retries: 40
start_period: 20s
deploy:
resources:
limits:
cpus: "${S3_CPU_LIMIT:-1.0}"
memory: "${S3_MEM_LIMIT:-768M}"
networks:
- backend
# One-shot, serialised schema migration. API replicas never run DDL.
migrate:
<<: *api-build
restart: "no"
environment:
DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env}
CACHE_URL: ${CACHE_URL:-redis://redis:6379/0}
DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3}
S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
command: ["python", "-m", "app.migrate"]
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: "256M"
networks:
- backend
# Idempotent private-bucket bootstrap with the administrative identity.
s3-init:
<<: *api-build
restart: "no"
environment:
DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env}
CACHE_URL: ${CACHE_URL:-redis://redis:6379/0}
S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333}
S3_REGION: ${S3_REGION:-us-east-1}
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env}
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env}
S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?set S3_ADMIN_ACCESS_KEY_ID in .env}
S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?set S3_ADMIN_SECRET_ACCESS_KEY in .env}
command: ["python", "-m", "app.s3_bootstrap"]
depends_on:
seaweedfs:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: "256M"
networks:
- backend
api:
<<: *api-build
restart: unless-stopped
environment: *api-environment
expose:
- "8080"
depends_on:
migrate:
condition: service_completed_successfully
s3-init:
condition: service_completed_successfully
redis:
condition: service_healthy
stop_grace_period: 15s
security_opt:
- no-new-privileges:true
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_LIMIT:-1.0}"
memory: "${API_MEM_LIMIT:-512M}"
networks:
- backend
lb:
image: nginx:1.27.5-alpine
restart: unless-stopped
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
ports:
- "${API_PORT:-8080}:8080"
healthcheck:
test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:8080/health/live >/dev/null"]
interval: 5s
timeout: 3s
retries: 10
deploy:
resources:
limits:
cpus: "${LB_CPU_LIMIT:-0.5}"
memory: "${LB_MEM_LIMIT:-256M}"
networks:
- backend
# Optional operator tool: docker compose --profile tools run --rm reconcile \
# python -m app.reconcile --apply
reconcile:
<<: *api-build
profiles: ["tools"]
restart: "no"
environment: *api-environment
command: ["python", "-m", "app.reconcile", "--dry-run"]
depends_on:
migrate:
condition: service_completed_successfully
seaweedfs:
condition: service_healthy
networks:
- backend
volumes:
pgdata:
redisdata:
seaweeddata:
networks:
backend:
driver: bridge
+37
View File
@@ -0,0 +1,37 @@
# Lab4 extensions: presigned direct upload (track 1)
The mandatory contract in `openapi.json` is unchanged. The proxy endpoint
`POST /api/v1/chats/{chat_id}/attachments` remains the required path and is the
fallback for clients that cannot reach S3. This extension adds an optional
direct path:
1. `POST /api/v1/chats/{chat_id}/attachments/presign` with
`{filename, content_type, size_bytes, sha256}` creates a `pending`
attachment row and returns a short-lived presigned `PUT` URL.
2. The client uploads the bytes directly to the private bucket using the URL
and the returned `Content-Type` / `x-amz-meta-sha256` headers.
3. `POST /api/v1/attachments/{attachment_id}/complete` with
`{sha256, size_bytes}` verifies ownership, the stored object size and the
SHA-256 of the bytes. Only then does the attachment become `ready`.
## Rules
- The bucket stays private. The URL is the only grant, expires after
`PRESIGN_TTL_SECONDS` (default 300 s), and uses a unique server-generated
object key.
- Complete is owner-only and chat-member-only. A different user receives 404.
- Complete is idempotent for the same `(sha256, size_bytes)` pair. A different
pair on a ready attachment is `409`.
- Completing before the object exists, after the URL expired, or after the
pending row was cleaned up is `409`.
- A wrong size or checksum is `400`; the wrong object is deleted and is never
promoted to `ready`.
- `GET /api/v1/attachments/{id}` may return `pending` for this extension.
The mandatory proxy path always returns `ready`.
- Incomplete uploads are removed by `python -m app.reconcile --apply` after
`PRESIGN_GRACE_SECONDS` (default 3600 s). Orphan objects that never got a
metadata row are removed after `ORPHAN_GRACE_SECONDS` (default 86400 s).
- The complete call reads the object once through the API to verify its
checksum. That is a deliberate integrity cost: direct upload avoids API
ingress, but the API still verifies egress. A future S3 checksum API could
remove this read; SeaweedFS 3.80 does not provide the same portable hook.
+158
View File
@@ -0,0 +1,158 @@
{
"openapi": "3.1.0",
"info": {
"title": "Messenger lab4 extensions: presigned direct upload",
"version": "1.4.0-extension.1",
"description": "Backward-compatible extension. The mandatory proxy upload in openapi.json remains supported and is the fallback."
},
"paths": {
"/api/v1/chats/{chat_id}/attachments/presign": {
"post": {
"operationId": "presignAttachmentUpload",
"summary": "Create a short-lived presigned PUT URL for a private-bucket object",
"parameters": [
{
"name": "chat_id",
"in": "path",
"required": true,
"schema": {"type": "string", "format": "uuid"}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/PresignRequest"}
}
}
},
"responses": {
"201": {
"description": "URL created; the attachment is pending until complete",
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/PresignResponse"}
}
}
},
"400": {"$ref": "#/components/responses/Error"},
"401": {"$ref": "#/components/responses/Error"},
"404": {"$ref": "#/components/responses/Error"},
"413": {"$ref": "#/components/responses/Error"},
"415": {"$ref": "#/components/responses/Error"},
"503": {"$ref": "#/components/responses/Error"}
}
}
},
"/api/v1/attachments/{attachment_id}/complete": {
"post": {
"operationId": "completeAttachmentUpload",
"summary": "Verify size and SHA-256 of a direct upload and make it ready",
"parameters": [
{
"name": "attachment_id",
"in": "path",
"required": true,
"schema": {"type": "string", "format": "uuid"}
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/CompleteRequest"}
}
}
},
"responses": {
"200": {
"description": "Ready attachment, or the same ready attachment on an idempotent retry",
"content": {
"application/json": {
"schema": {"$ref": "#/components/schemas/AttachmentExtension"}
}
}
},
"400": {"$ref": "#/components/responses/Error"},
"401": {"$ref": "#/components/responses/Error"},
"404": {"$ref": "#/components/responses/Error"},
"409": {"$ref": "#/components/responses/Error"},
"503": {"$ref": "#/components/responses/Error"}
}
}
}
},
"components": {
"responses": {
"Error": {
"description": "Typed error",
"content": {
"application/json": {
"schema": {"$ref": "../openapi.json#/components/schemas/Error"}
}
}
}
},
"schemas": {
"PresignRequest": {
"type": "object",
"properties": {
"filename": {"type": "string", "minLength": 1, "maxLength": 255},
"content_type": {"type": "string", "enum": ["image/png", "image/jpeg"]},
"size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760},
"sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}
},
"required": ["filename", "content_type", "size_bytes", "sha256"],
"additionalProperties": false
},
"PresignResponse": {
"type": "object",
"properties": {
"attachment_id": {"type": "string", "format": "uuid"},
"upload_url": {"type": "string", "format": "uri"},
"method": {"type": "string", "const": "PUT"},
"headers": {
"type": "object",
"properties": {
"Content-Type": {"type": "string", "enum": ["image/png", "image/jpeg"]},
"x-amz-meta-sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}
},
"required": ["Content-Type", "x-amz-meta-sha256"],
"additionalProperties": false
},
"expires_at": {"type": "string", "format": "date-time"}
},
"required": ["attachment_id", "upload_url", "method", "headers", "expires_at"],
"additionalProperties": false
},
"CompleteRequest": {
"type": "object",
"properties": {
"sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
"size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760}
},
"required": ["sha256", "size_bytes"],
"additionalProperties": false
},
"AttachmentExtension": {
"type": "object",
"properties": {
"id": {"type": "string", "format": "uuid"},
"chat_id": {"type": "string", "format": "uuid"},
"owner_id": {"type": "string", "format": "uuid"},
"filename": {"type": "string", "minLength": 1, "maxLength": 255},
"content_type": {"type": "string", "enum": ["image/png", "image/jpeg"]},
"size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760},
"sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"},
"status": {"type": "string", "enum": ["pending", "ready", "failed"]},
"variants": {"type": "array", "items": {"type": "object"}},
"error_code": {"type": ["string", "null"]},
"created_at": {"type": "string", "format": "date-time"},
"updated_at": {"type": "string", "format": "date-time"}
},
"required": ["id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "status", "variants", "error_code", "created_at", "updated_at"],
"additionalProperties": false
}
}
}
}
+22 -2
View File
@@ -1,5 +1,25 @@
# Доказательства
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Небольшие обезличенные результаты запусков на локальном учебном стенде
(Windows + Docker Desktop, Linux containers; API limit 512 MiB, SeaweedFS
limit 768 MiB). Дата: 2026-11-07. Секреты, токены, cookies, signed URLs и
raw dumps в файлы не включались.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
| Файл | Что доказывает |
| --- | --- |
| `check.txt` | offline-проверка структуры и синтаксиса |
| `smoke.txt` | публичный smoke lab4 |
| `attachments-negative.txt` | лимиты, MIME, повреждение, filename, чужой id/чат |
| `upload-memory.json`, `upload-memory.txt` | 10 параллельных загрузок и пик RAM |
| `transfer-compare.json`, `transfer-compare.txt` | proxy vs presigned direct: трафик/RAM на двух размерах |
| `presigned-flow.txt` | presigned PUT, complete, checksum, owner-only, negative |
| `private-bucket.txt` | private bucket, ограниченные credentials, закрытые admin UI |
| `s3-init-idempotent.txt` | повторный запуск bootstrap видит существующий bucket |
| `reconcile.txt` | dry-run/apply, grace period, orphan и stale pending |
| `persistence.txt` | restart с volumes сохраняет данные и сессию |
| `failover.txt` | остановка одной реплики, восстановление и сохранность |
| `instance-spread.txt`, `idempotency-race.txt` | регрессия lab3: ≥2 реплики, общий Idempotency-Key |
| `load.json` | closed-loop нагрузка 30 s |
| `compose-ps.txt` | состав и healthcheck'и стенда |
Команды запуска и интерпретация — в `REPORT.md` и `student-tests/README.md`.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 4,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.069,
"concurrency": 4,
"successful_requests": 6789,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 225.781,
"successful_latency_ms": {
"p50": 14.836,
"p95": 33.787,
"p99": 66.655
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+39
View File
@@ -0,0 +1,39 @@
{
"container_prefix": "mtusi-lab4-api",
"sizes": [
{
"direct": {
"api_memory_growth_mib": 0.0,
"api_memory_peak_mib": 153.66,
"api_network_bytes": 564368,
"client_to_api_bytes": 0,
"client_to_s3_bytes": 552409,
"file_bytes": 552409
},
"proxy": {
"api_memory_growth_mib": 0.0,
"api_network_bytes": 1120316,
"client_to_api_bytes": 552409,
"client_to_s3_bytes": 0,
"file_bytes": 552409
}
},
{
"direct": {
"api_memory_growth_mib": 0.0,
"api_memory_peak_mib": 155.41,
"api_network_bytes": 4446591,
"client_to_api_bytes": 0,
"client_to_s3_bytes": 4424547,
"file_bytes": 4424547
},
"proxy": {
"api_memory_growth_mib": 1.4,
"api_network_bytes": 8888758,
"client_to_api_bytes": 4424547,
"client_to_s3_bytes": 0,
"file_bytes": 4424547
}
}
]
}
Binary file not shown.
+18
View File
@@ -0,0 +1,18 @@
{
"baseline_mib": {
"mtusi-lab4-api-1": 83.17,
"mtusi-lab4-api-2": 85.38
},
"elapsed_seconds": 1.697,
"file_bytes": 4431995,
"peak_growth_mib": {
"mtusi-lab4-api-1": 22.53,
"mtusi-lab4-api-2": 22.52
},
"peak_mib": {
"mtusi-lab4-api-1": 105.7,
"mtusi-lab4-api-2": 107.9
},
"successful_uploads": 10,
"workers": 10
}
Binary file not shown.
+57
View File
@@ -0,0 +1,57 @@
# Public entry point for laboratory 4.
#
# Docker's embedded DNS (127.0.0.11) returns every API replica IP for the
# Compose service name "api". The shared upstream zone + "resolve" picks up
# scale-up/scale-down and replacement containers without a reload. Passive
# checks remove a replica after failed connections; proxy_next_upstream lets a
# safe retry reach another backend.
worker_processes auto;
events {
worker_connections 4096;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
server_tokens off;
access_log /dev/stdout;
error_log /dev/stderr warn;
resolver 127.0.0.11 valid=2s ipv6=off;
upstream api_backend {
zone api_backend 64k;
server api:8080 resolve max_fails=3 fail_timeout=2s;
keepalive 32;
}
server {
listen 8080;
# The API enforces 10 MiB; leave a little room for HTTP framing.
client_max_body_size 12m;
location / {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 2s;
proxy_send_timeout 60s;
proxy_read_timeout 60s;
# Do not retry non-idempotent requests transparently. The client
# repeats an uncertain POST with the same Idempotency-Key, which is
# the contract-level guarantee across replicas. Application 503
# responses pass through unchanged.
proxy_next_upstream error timeout http_502 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 3s;
}
}
}
+49
View File
@@ -0,0 +1,49 @@
#!/bin/sh
# Generate the SeaweedFS S3 identity file from local Compose secrets, then run
# one master + volume + filer + S3 process. The file lives only in /tmp; the
# repository never contains credentials.
set -eu
: "${S3_ADMIN_ACCESS_KEY_ID:?missing S3_ADMIN_ACCESS_KEY_ID}"
: "${S3_ADMIN_SECRET_ACCESS_KEY:?missing S3_ADMIN_SECRET_ACCESS_KEY}"
: "${S3_ACCESS_KEY_ID:?missing S3_ACCESS_KEY_ID}"
: "${S3_SECRET_ACCESS_KEY:?missing S3_SECRET_ACCESS_KEY}"
case "$S3_ADMIN_ACCESS_KEY_ID$S3_ADMIN_SECRET_ACCESS_KEY$S3_ACCESS_KEY_ID$S3_SECRET_ACCESS_KEY" in
*[!A-Za-z0-9._-]*)
echo "S3 credentials must contain only A-Z a-z 0-9 . _ -" >&2
exit 1
;;
esac
umask 077
cat > /tmp/s3.json <<EOF
{
"identities": [
{
"name": "admin",
"credentials": [
{"accessKey": "$S3_ADMIN_ACCESS_KEY_ID", "secretKey": "$S3_ADMIN_SECRET_ACCESS_KEY"}
],
"actions": ["Admin", "Read", "Write", "List", "Tagging"]
},
{
"name": "app",
"credentials": [
{"accessKey": "$S3_ACCESS_KEY_ID", "secretKey": "$S3_SECRET_ACCESS_KEY"}
],
"actions": ["Read", "Write", "List"]
}
]
}
EOF
exec /usr/bin/weed -logtostderr=true server \
-dir=/data \
-volume.max=0 \
-master.volumePreallocate \
-master.volumeSizeLimitMB=1024 \
-filer \
-s3 \
-s3.port=8333 \
-s3.config=/tmp/s3.json
+51
View File
@@ -0,0 +1,51 @@
-- Laboratory 2 initial schema.
-- PostgreSQL is the source of truth for users, chats, membership and messages.
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100)
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100)
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000)
);
-- Reverse membership lookup: "my chats, newest last".
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
ON chat_members (user_id, chat_id);
-- Chat page ordering and keyset pagination.
CREATE INDEX IF NOT EXISTS chats_created_at_id_idx
ON chats (created_at, id);
-- The only history access path:
-- WHERE chat_id = ? AND (created_at, id) > (?, ?)
-- ORDER BY created_at, id LIMIT ?
-- It supports the range scan without reading the whole chat history.
CREATE INDEX IF NOT EXISTS messages_chat_created_at_id_idx
ON messages (chat_id, created_at, id);
+26
View File
@@ -0,0 +1,26 @@
-- Laboratory 3: shared idempotency for message creation.
-- Scope is (user_id, method, path, key); the stored result is the message id.
-- The row and the message are written in one transaction by the API.
CREATE TABLE IF NOT EXISTS idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method text NOT NULL,
path text NOT NULL,
key text NOT NULL,
request_hash text NOT NULL,
message_id uuid REFERENCES messages(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, method, path, key),
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT idempotency_method_post CHECK (method = 'POST')
);
-- Expired rows are removed lazily; this index keeps that cleanup bounded.
CREATE INDEX IF NOT EXISTS idempotency_expires_at_idx
ON idempotency_keys (expires_at);
-- The message id is unique per stored key, but the same message is never
-- shared across keys; a plain lookup by primary key is enough for replay.
CREATE INDEX IF NOT EXISTS idempotency_message_id_idx
ON idempotency_keys (message_id);
+62
View File
@@ -0,0 +1,62 @@
-- Laboratory 4: attachment metadata in PostgreSQL, bytes in a private S3 bucket.
--
-- The API assigns the object key. The original filename is display metadata
-- only and is never used to build a key. A row is inserted only after the S3
-- PUT succeeds; the proxy upload compensates a failed INSERT by deleting the
-- object, and the reconciler cleans up objects left by a process crash.
CREATE TABLE IF NOT EXISTS attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 text NOT NULL,
object_key text NOT NULL,
status text NOT NULL DEFAULT 'ready',
error_code text,
presign_expires_at timestamptz,
missing_since timestamptz,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
CONSTRAINT attachments_content_type_check
CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size_check CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_check CHECK (sha256 ~ '^[a-f0-9]{64}$'),
CONSTRAINT attachments_status_check
CHECK (status IN ('pending', 'ready', 'failed')),
CONSTRAINT attachments_filename_check
CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_error_code_check
CHECK (error_code IS NULL OR error_code IN
('invalid_image', 'processing_failed', 'retry_exhausted', 'missing_object'))
);
CREATE INDEX IF NOT EXISTS attachments_chat_created_idx
ON attachments (chat_id, created_at, id);
-- Reconciler access paths.
CREATE INDEX IF NOT EXISTS attachments_status_created_idx
ON attachments (status, created_at);
CREATE INDEX IF NOT EXISTS attachments_object_key_idx
ON attachments (object_key);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
position integer NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position),
CONSTRAINT message_attachments_position_check CHECK (position >= 0 AND position < 10)
);
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
ON message_attachments (attachment_id);
-- A message may carry an empty text only when it has at least one attachment.
-- The API enforces the semantic part; this constraint is the storage backstop.
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_length;
ALTER TABLE messages
ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000);
+10
View File
@@ -0,0 +1,10 @@
-- The idempotency reservation is inserted before the message row inside one
-- transaction. A deferred foreign key lets both statements commit together
-- while still guaranteeing that a committed key always points at a message.
ALTER TABLE idempotency_keys
DROP CONSTRAINT IF EXISTS idempotency_keys_message_id_fkey;
ALTER TABLE idempotency_keys
ADD CONSTRAINT idempotency_keys_message_id_fkey
FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE
DEFERRABLE INITIALLY DEFERRED;
+4 -1
View File
@@ -1,2 +1,5 @@
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
# Full OpenAPI/JSON Schema validation and host-side student checks.
openapi-spec-validator==0.7.2
jsonschema==4.23.0
Pillow==11.0.0
boto3==1.35.76
+7
View File
@@ -0,0 +1,7 @@
Flask==3.1.0
gunicorn==23.0.0
psycopg[binary,pool]==3.2.3
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.35.76
Pillow==11.0.0
+10
View File
@@ -0,0 +1,10 @@
@echo off
REM Restart API and ALL state stores without deleting named volumes.
setlocal
cd /d "%~dp0.."
if not exist .env (
echo missing .env; copy .env.example and set local synthetic credentials>&2
exit /b 2
)
docker compose --env-file .env restart postgres redis seaweedfs api lb
exit /b %ERRORLEVEL%
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
# Restart API and ALL state stores without deleting named volumes.
# PostgreSQL and Redis keep their volumes; the public persistence test polls
# /health/ready itself, so this hook only performs the restart and returns
# non-zero on a Compose error.
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
if [ ! -f .env ]; then
echo "missing .env; copy .env.example and set local synthetic credentials" >&2
exit 2
fi
# Do not restart the one-shot migrate service here: the schema is already
# persisted in the PostgreSQL volume. Restart every long-running component.
docker compose --env-file .env restart postgres redis seaweedfs api lb
+43
View File
@@ -0,0 +1,43 @@
#!/bin/sh
# Equivalent of a rolling update for Docker Compose: replace one API replica
# at a time, keeping the service at the configured replica count. Run
# `docker compose build api` first when testing a new image. No volumes,
# database rows or sessions are touched.
set -eu
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
COMPOSE="docker compose --env-file .env"
COUNT=${API_REPLICAS:-2}
wait_for_healthy() {
deadline=$(( $(date +%s) + 90 ))
while :; do
ids=$($COMPOSE ps -q api)
running=0
healthy=0
for id in $ids; do
state=$(docker inspect --format '{{.State.Status}}' "$id")
health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id")
if [ "$state" = "running" ]; then running=$((running + 1)); fi
if [ "$health" = "healthy" ] || [ "$health" = "none" ]; then healthy=$((healthy + 1)); fi
done
if [ "$running" -ge "$COUNT" ] && [ "$healthy" -ge "$COUNT" ]; then
return 0
fi
if [ "$(date +%s)" -ge "$deadline" ]; then
echo "rolling-update: replicas did not become healthy" >&2
return 1
fi
sleep 1
done
}
wait_for_healthy
for id in $($COMPOSE ps -q api); do
echo "replacing $id"
docker stop --time 15 "$id" >/dev/null
docker rm "$id" >/dev/null
$COMPOSE up -d --no-deps --no-build --scale api="$COUNT" api
wait_for_healthy
done
echo "rolling-update: all $COUNT replicas replaced"
+8
View File
@@ -0,0 +1,8 @@
# Stop exactly ONE API replica from a Windows host.
# The public failover test sets TARGET_INSTANCE_ID; the PowerShell helper maps
# the alias (container hostname) to the matching Compose container.
@echo off
setlocal
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1"
exit /b %ERRORLEVEL%
+21
View File
@@ -0,0 +1,21 @@
$ErrorActionPreference = 'Stop'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$target = $env:TARGET_INSTANCE_ID
if ([string]::IsNullOrWhiteSpace($target)) { exit 2 }
$ids = @(docker compose --env-file .env ps -q api)
foreach ($id in $ids) {
if ([string]::IsNullOrWhiteSpace($id)) { continue }
$hostName = (docker inspect --format '{{.Config.Hostname}}' $id).Trim()
$containerName = (docker inspect --format '{{.Name}}' $id).Trim().TrimStart('/')
if ($hostName -eq $target -or $hostName.StartsWith($target) -or
$target.StartsWith($hostName) -or $containerName -eq $target) {
docker stop --time 5 $id | Out-Null
exit 0
}
}
Write-Error 'stop-one: no API replica matches TARGET_INSTANCE_ID'
exit 3
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Stop exactly ONE API replica identified by the public X-Instance-Id alias.
# The public failover test sets TARGET_INSTANCE_ID and expects this script to
# return within 10 seconds while leaving the replica stopped. It never touches
# PostgreSQL, Redis, the LB or named volumes.
set -eu
: "${TARGET_INSTANCE_ID:?test provides the target API instance alias}"
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT"
ENV_FILE=${ENV_FILE:-.env}
COMPOSE="docker compose --env-file $ENV_FILE"
target=$TARGET_INSTANCE_ID
found=''
for id in $($COMPOSE ps -q api); do
host=$(docker inspect --format '{{.Config.Hostname}}' "$id")
name=$(docker inspect --format '{{.Name}}' "$id")
case "$host" in
"$target"|"$target"*) found=$id; break ;;
esac
case "$target" in
"$host"|"$host"*) found=$id; break ;;
esac
# INSTANCE_ID may be set explicitly by an operator; then it is not the
# container hostname and the default mapping cannot be used.
if [ "$host" != "$name" ] && [ "${name#/}" = "$target" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
echo "stop-one: no API replica matches TARGET_INSTANCE_ID" >&2
exit 3
fi
# -t 5 bounds the graceful drain; gunicorn exits earlier when idle. Docker
# does not restart a container stopped explicitly, even with restart policy.
docker stop --time 5 "$found" >/dev/null
exit 0
+68
View File
@@ -0,0 +1,68 @@
# Студенческие проверки lab4
Все проверки, кроме `rollback_chat.py`, `explain_history.py` и `pool_settings.py`,
запускаются на хосте и обращаются к API через LB. Три проверки используют `psycopg`
и запускаются внутри одного контейнера API. Для новых проверок нужны `Pillow` и
`boto3` (см. `requirements-dev.txt`).
```sh
# 1. Одна сессия обслуживается минимум двумя реальными репликами
python student-tests/instance_spread.py --base-url http://localhost:8080
# 2. Конкурентный повтор одного Idempotency-Key через разные реплики
python student-tests/idempotency_race.py --base-url http://localhost:8080 --workers 20
# 3. Негативная аутентификация и logout (перенесено из lab2)
python student-tests/auth_negative.py --base-url http://localhost:8080
# 4. Гонка регистрации одного username (перенесено из lab2)
python student-tests/race_username.py --base-url http://localhost:8080 --workers 8
# 5. Откат составной операции (внутри контейнера; --index обязателен при replicas>1)
docker compose exec -T --index 1 api python student-tests/rollback_chat.py
# 6. EXPLAIN на 10 000 сообщений (внутри контейнера)
docker compose exec -T --index 1 api python student-tests/explain_history.py \
--output /tmp/explain_history.txt
# 7. Пул и SQL-таймауты (внутри контейнера)
docker compose exec -T --index 1 api python student-tests/pool_settings.py
# Lab4: негативные сценарии вложений
python student-tests/attachments_negative.py --base-url http://localhost:8080
# Lab4: 10 одновременных загрузок и пик RAM API
python student-tests/upload_memory.py --base-url http://localhost:8080 \
--output evidence/upload-memory.json
# Lab4/5: presigned direct upload и обязательный proxy fallback
python student-tests/presigned_flow.py --base-url http://localhost:8080
# Lab5 track 1: сравнение трафика/RAM API proxy vs direct на двух размерах
python student-tests/transfer_compare.py --base-url http://localhost:8080 \
--output evidence/transfer-compare.json
# Lab4: приватный bucket и отсутствие опубликованных admin UI
python student-tests/private_bucket.py --base-url http://localhost:8080
# Lab4/5: reconciler, grace period, очистка incomplete upload
python student-tests/reconcile_orphans.py --base-url http://localhost:8080
```
`instance_spread.py` проверяет не только заголовок, но и то, что запись, принятая
одной репликой, читается через другую. `idempotency_race.py` отправляет один ключ
параллельно, проверяет ровно один Message, 409 на другой body, 400 на отсутствующий
ключ и независимость scope по пользователю и чату.
`attachments_negative.py` проверяет лимит 10 MiB, пустое/повреждённое изображение,
подмену MIME, лимит 40 млн пикселей, опасные имена файлов, чужой attachment_id и
вложение из другого чата. `upload_memory.py` измеряет пик памяти контейнеров API
через `docker stats` при 10 параллельных загрузках. `presigned_flow.py` проверяет
короткую подпись, complete, checksum, owner-only, идемпотентность и отрицательные
сценарии. `reconcile_orphans.py` внедряет orphan-объект и незавершённую загрузку,
показывает dry-run, безопасный grace period и повторный `--apply`.
`fault_matrix.py` из lab2 не включён в основной набор: при отказе PostgreSQL/Redis
nginx исключает нездоровые API-реплики и сам возвращает 502/503 без JSON-тела,
что является корректным поведением LB, но не проверкой формата ошибки API.
Проверка отказов зависимостей для lab4 описана в `REPORT.md`.
+176
View File
@@ -0,0 +1,176 @@
#!/usr/bin/env python3
"""Lab4 negative attachment checks: size, corruption, MIME, ownership, chat scope."""
from __future__ import annotations
import argparse
import io
import struct
import uuid
import zlib
from urllib.parse import quote
from PIL import Image
from common import HttpClient, assert_status, error_code
MAX_UPLOAD = 10_485_760
def _png_chunk(kind: bytes, data: bytes) -> bytes:
return (
len(data).to_bytes(4, "big")
+ kind
+ data
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
)
def png_bytes(width: int = 320, height: int = 200) -> bytes:
buffer = io.BytesIO()
Image.new("RGB", (width, height), (30, 120, 200)).save(buffer, format="PNG")
return buffer.getvalue()
def jpeg_bytes(width: int = 320, height: int = 200) -> bytes:
buffer = io.BytesIO()
Image.new("RGB", (width, height), (200, 80, 30)).save(buffer, format="JPEG")
return buffer.getvalue()
def png_header(width: int, height: int) -> bytes:
"""Valid signature/IHDR/IEND; enough for the pixel budget check, no decode."""
return (
b"\x89PNG\r\n\x1a\n"
+ _png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ _png_chunk(b"IEND", b"")
)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_reg = client.register()
bob, bob_reg = client.register()
eve, _eve_reg = client.register()
alice_token = client.login(alice_reg)["token"]
bob_token = client.login(bob_reg)["token"]
eve_token = client.login(_eve_reg)["token"]
status, _h, chat, _r = client.call(
"POST",
"/api/v1/chats",
body={"title": "Attachment negatives", "member_ids": [alice["id"], bob["id"]]},
token=alice_token,
)
assert_status(status, 201, "create chat")
status, _h, other_chat, _r = client.call(
"POST",
"/api/v1/chats",
body={"title": "Another chat", "member_ids": [alice["id"], bob["id"]]},
token=alice_token,
)
assert_status(status, 201, "create other chat")
def upload(content: bytes, content_type: str, filename: str, token=alice_token, chat_id=None):
path = (
"/api/v1/chats/"
+ (chat_id or chat["id"])
+ "/attachments?filename="
+ quote(filename, safe="")
)
return client.raw(
"POST",
path,
data=content,
headers={"Authorization": "Bearer " + token, "Content-Type": content_type},
)
def expect(result, expected: int, label: str) -> dict:
status, headers, payload, _raw = result
assert_status(status, expected, label)
if status >= 400:
if not isinstance(payload, dict) or payload.get("error", {}).get("code") in (None, "internal_error"):
raise AssertionError(f"{label}: expected a typed user error, got {payload!r}")
if payload["error"]["request_id"] != headers.get("X-Request-Id"):
raise AssertionError(f"{label}: request_id mismatch")
return payload
png = png_bytes()
jpeg = jpeg_bytes()
attachment = expect(upload(png, "image/png", "ok.png"), 201, "valid PNG")
if attachment["status"] != "ready":
raise AssertionError("valid PNG did not become ready")
expect(
upload(b"\x89PNG\r\n\x1a\n" + b"x" * (MAX_UPLOAD + 1), "image/png", "big.png"),
413,
"oversized body",
)
expect(upload(b"", "image/png", "empty.png"), 415, "empty body")
expect(upload(b"\x89PNG\r\n\x1a\nnot-an-image", "image/png", "broken.png"), 415, "corrupted PNG")
expect(upload(jpeg, "image/png", "spoof.png"), 415, "JPEG bytes declared as PNG")
expect(upload(png, "image/jpeg", "spoof.jpg"), 415, "PNG bytes declared as JPEG")
expect(upload(png, "text/plain", "plain.txt"), 415, "unsupported content type")
expect(upload(png_header(10_000, 5_000), "image/png", "huge.png"), 413, "pixel budget")
expect(upload(png, "image/png", "../escape.png"), 400, "path traversal filename")
expect(upload(png, "image/png", "a/b.png"), 400, "slash filename")
expect(upload(png, "image/png", "a\\b.png"), 400, "backslash filename")
# A member of the chat still gets 404 for an unknown attachment id.
missing_id = str(uuid.uuid4())
status, _h, payload, _r = client.call(
"GET", "/api/v1/attachments/" + missing_id, token=bob_token
)
assert_status(status, 404, "unknown attachment metadata")
if error_code(payload) == "internal_error":
raise AssertionError("unknown attachment leaked a 500")
# A non-member cannot read metadata or bytes.
for suffix in ("", "/content"):
status, _h, payload, _r = client.call(
"GET", "/api/v1/attachments/" + attachment["id"] + suffix, token=eve_token
)
assert_status(status, 404, "outsider attachment read")
if error_code(payload) == "internal_error":
raise AssertionError("outsider read leaked a 500")
# Uploading into another chat and linking it here must be a 404, not a 500.
foreign = expect(upload(png, "image/png", "foreign.png", chat_id=other_chat["id"]), 201, "other-chat upload")
status, _h, payload, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
body={"text": "", "attachment_ids": [foreign["id"]]},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
assert_status(status, 404, "attachment from another chat")
if error_code(payload) == "internal_error":
raise AssertionError("cross-chat attachment leaked a 500")
# Duplicate and too-many ids are request errors, not 500.
status, _h, _p, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
body={"text": "", "attachment_ids": [attachment["id"], attachment["id"]]},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
assert_status(status, 400, "duplicate attachment ids")
status, _h, _p, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
body={"text": "", "attachment_ids": [attachment["id"]] * 11},
token=alice_token,
headers={"Idempotency-Key": uuid.uuid4().hex},
)
assert_status(status, 400, "too many attachment ids")
print("PASS: lab4 attachment negatives (size, corruption, MIME, filename, ownership, chat scope)")
if __name__ == "__main__":
main()
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Wrong password, Basic-on-business-endpoint, logout and reuse checks."""
from __future__ import annotations
import argparse
import base64
from common import HttpClient, assert_status, error_code
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
client = HttpClient(args.base_url)
user, registration = client.register()
wrong = base64.b64encode(
(registration["username"] + ":wrong-password-value").encode("utf-8")
).decode("ascii")
status, headers, payload, _ = client.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + wrong}
)
assert_status(status, 401, "wrong password")
assert error_code(payload) == "unauthorized"
assert headers.get("WWW-Authenticate", "").lower().startswith("basic")
good = base64.b64encode(
(registration["username"] + ":" + registration["password"]).encode("utf-8")
).decode("ascii")
status, headers, _, _ = client.call(
"GET", "/api/v1/users/me", headers={"Authorization": "Basic " + good}
)
assert_status(status, 401, "Basic on business endpoint")
assert headers.get("WWW-Authenticate", "").lower().startswith("bearer")
session = client.login(registration)
token = session["token"]
status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 200, "login")
assert me["id"] == user["id"]
status, _, _, raw = client.call("DELETE", "/api/v1/auth/sessions/current", token=token)
assert_status(status, 204, "logout")
assert raw == b"", "204 must have an empty body"
status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 401, "token after logout")
assert error_code(payload) == "unauthorized"
assert headers.get("WWW-Authenticate", "").lower().startswith("bearer")
status, _, _, _ = client.call("DELETE", "/api/v1/auth/sessions/current", token=token)
assert_status(status, 401, "second logout")
print("PASS: wrong password, Basic rejection, logout and token reuse checks")
if __name__ == "__main__":
main()
+96
View File
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Small standard-library HTTP helpers for the student checks."""
from __future__ import annotations
import base64
import json
import os
import uuid
import urllib.error
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/")
class HttpClient:
def __init__(self, base_url: str | None = None):
self.base = (base_url or BASE_URL).rstrip("/")
def call(self, method: str, path: str, body=None, token: str | None = None, headers=None):
h = {"Accept": "application/json"}
data = None
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
h["Content-Type"] = "application/json"
if token:
h["Authorization"] = "Bearer " + token
h.update(headers or {})
request = urllib.request.Request(self.base + path, data=data, headers=h, method=method)
try:
response = urllib.request.urlopen(request, timeout=10)
except urllib.error.HTTPError as error:
response = error
with response:
raw = response.read()
payload = None
if raw:
try:
payload = json.loads(raw)
except ValueError:
payload = None
return response.code, response.headers, payload, raw
def raw(self, method: str, path: str, data: bytes | None = None, headers=None, timeout=60):
"""Send raw bytes (uploads, presigned PUT) and return status/headers/json/raw."""
h = {"Accept": "application/json"}
h.update(headers or {})
url = path if path.startswith(("http://", "https://")) else self.base + path
request = urllib.request.Request(url, data=data, headers=h, method=method)
try:
response = urllib.request.urlopen(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
raw = response.read()
payload = None
if raw:
try:
payload = json.loads(raw)
except ValueError:
payload = None
return response.code, response.headers, payload, raw
def register(self) -> tuple[dict, dict]:
suffix = uuid.uuid4().hex[:12]
registration = {
"username": "st_" + suffix,
"display_name": "Студент " + suffix,
"password": "T3st-" + uuid.uuid4().hex,
}
status, _headers, user, _raw = self.call("POST", "/api/v1/users", registration)
if status != 201:
raise AssertionError(f"registration failed: HTTP {status}")
return user, registration
def login(self, registration: dict) -> dict:
credential = base64.b64encode(
(registration["username"] + ":" + registration["password"]).encode("utf-8")
).decode("ascii")
status, _headers, session, _raw = self.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credential}
)
if status != 201:
raise AssertionError(f"login failed: HTTP {status}")
return session
def assert_status(actual: int, expected: int, label: str) -> None:
if actual != expected:
raise AssertionError(f"{label}: expected HTTP {expected}, received {actual}")
def error_code(payload) -> str | None:
if not isinstance(payload, dict):
return None
error = payload.get("error")
return error.get("code") if isinstance(error, dict) else None
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Expiry check: run against an API with SESSION_TTL_SECONDS=2 (or pass --wait)."""
from __future__ import annotations
import argparse
import time
from common import HttpClient, assert_status, error_code
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--wait", type=float, default=3.0)
args = parser.parse_args()
client = HttpClient(args.base_url)
user, registration = client.register()
session = client.login(registration)
token = session["token"]
status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 200, "fresh session")
assert payload["id"] == user["id"], "wrong current user"
time.sleep(args.wait)
status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 401, "expired session")
assert error_code(payload) == "unauthorized", "expired session must be unauthorized"
assert headers.get("WWW-Authenticate", "").lower().startswith("bearer"), "missing Bearer challenge"
print("PASS: expired session is rejected with 401 and Bearer challenge")
if __name__ == "__main__":
main()
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Create 10 000 synthetic messages and record EXPLAIN (ANALYZE, BUFFERS).
Run inside the API container (psycopg is installed there):
docker compose exec -T api python student-tests/explain_history.py \
--output /app/evidence/explain_history.txt
The evidence directory is mounted read-only in the container, so the example
below is normally executed with a writable bind mount or copied out.
"""
from __future__ import annotations
import argparse
import os
import uuid
from datetime import datetime, timezone
from pathlib import Path
import psycopg
MESSAGES = 10_000
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL"))
parser.add_argument("--output", default="evidence/explain_history.txt")
parser.add_argument("--messages", type=int, default=MESSAGES)
args = parser.parse_args()
if not args.database_url:
raise SystemExit("DATABASE_URL is required")
tag = uuid.uuid4().hex[:10]
user_id = uuid.uuid4()
chat_id = uuid.uuid4()
with psycopg.connect(args.database_url) as conn:
conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s)",
(user_id, "perf_" + tag, "Performance synthetic", "not-a-real-password-hash"),
)
conn.execute("INSERT INTO chats (id, title) VALUES (%s, %s)", (chat_id, "perf-" + tag))
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat_id, user_id)
)
conn.execute(
"""
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), %s, %s, 'synthetic message ' || g,
timestamptz '2026-01-01 00:00:00+00' + (g || ' milliseconds')::interval
FROM generate_series(1, %s) AS g
""",
(chat_id, user_id, args.messages),
)
total = conn.execute(
"SELECT count(*) FROM messages WHERE chat_id = %s", (chat_id,)
).fetchone()[0]
conn.execute("ANALYZE messages")
cursor = conn.execute(
"""
SELECT created_at, id FROM messages
WHERE chat_id = %s ORDER BY created_at, id
OFFSET %s LIMIT 1
""",
(chat_id, max(0, args.messages - 500)),
).fetchone()
plan_rows = conn.execute(
"""
EXPLAIN (ANALYZE, BUFFERS)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = %s AND (created_at, id) > (%s, %s)
ORDER BY created_at, id
LIMIT 51
""",
(chat_id, cursor[0], cursor[1]),
).fetchall()
conn.commit()
plan = "\n".join(row[0] for row in plan_rows)
if "messages_chat_created_at_id_idx" not in plan:
raise AssertionError("history query did not use the keyset index:\n" + plan)
if "Seq Scan" in plan:
raise AssertionError("history query performed a sequential scan:\n" + plan)
header = (
"# EXPLAIN (ANALYZE, BUFFERS) for keyset history pagination\n"
f"generated_at_utc: {datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')}\n"
f"messages_in_chat: {total}\n"
f"chat_id: {chat_id}\n"
"query: WHERE chat_id = ? AND (created_at, id) > (?, ?) "
"ORDER BY created_at, id LIMIT 51\n\n"
)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(header + plan + "\n", encoding="utf-8")
print(header + plan)
print(f"PASS: index used on {total} messages; evidence written to {output}")
if __name__ == "__main__":
main()
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env python3
"""Dependency-failure matrix: Redis and PostgreSQL outages must be bounded.
The check uses the local Compose project. It stops one dependency, verifies
503 (never 200 and never a trusted anonymous request), starts it again, waits
for readiness and confirms the original bearer session still works after the
Redis restart (AOF persistence).
"""
from __future__ import annotations
import argparse
import os
import subprocess
import time
from pathlib import Path
from common import HttpClient, assert_status, error_code
ROOT = Path(__file__).resolve().parents[1]
def compose(*args: str) -> None:
subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
check=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def wait_ready(client: HttpClient, expected: int, seconds: float = 90) -> None:
deadline = time.monotonic() + seconds
last = None
while time.monotonic() < deadline:
try:
status, _, _, _ = client.call("GET", "/health/ready")
last = status
if status == expected:
return
except OSError:
last = "connection error"
time.sleep(0.5)
raise AssertionError(f"readiness did not become {expected}; last={last}")
def expect_503(client: HttpClient, path: str, token: str | None, label: str) -> None:
status, _headers, payload, _raw = client.call("GET", path, token=token)
if status != 503:
raise AssertionError(f"{label}: expected 503, received {status}")
if error_code(payload) != "unavailable":
raise AssertionError(f"{label}: expected unavailable error code")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
client = HttpClient(args.base_url)
user, registration = client.register()
session = client.login(registration)
token = session["token"]
results = []
try:
compose("stop", "redis")
expect_503(client, "/health/ready", None, "Redis down readiness")
expect_503(client, "/api/v1/users/me", token, "Redis down authenticated request")
results.append("redis down: readiness=503, bearer request=503")
started = time.monotonic()
compose("start", "redis")
wait_ready(client, 200)
redis_recovery = round(time.monotonic() - started, 2)
status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 200, "session after Redis restart")
if me["id"] != user["id"]:
raise AssertionError("session identity changed after Redis restart")
results.append(f"redis up: readiness=200 in {redis_recovery}s, same token=200 (AOF persistence)")
compose("stop", "postgres")
time.sleep(1)
expect_503(client, "/health/ready", None, "PostgreSQL down readiness")
expect_503(client, "/api/v1/users/me", token, "PostgreSQL down authenticated request")
results.append("postgres down: readiness=503, bearer request=503")
started = time.monotonic()
compose("start", "postgres")
wait_ready(client, 200)
postgres_recovery = round(time.monotonic() - started, 2)
status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token)
assert_status(status, 200, "session after PostgreSQL restart")
results.append(f"postgres up: readiness=200 in {postgres_recovery}s, same token=200")
finally:
compose("start", "redis")
compose("start", "postgres")
for line in results:
print("OBSERVED:", line)
print("PASS: dependency loss produced only bounded 503 responses; no anonymous trust")
if __name__ == "__main__":
main()
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env python3
"""Concurrent shared-key replay across replicas plus scope/conflict checks."""
from __future__ import annotations
import argparse
import uuid
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from common import HttpClient
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--workers", type=int, default=20)
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_registration = client.register()
bob, bob_registration = client.register()
alice_token = client.login(alice_registration)["token"]
bob_token = client.login(bob_registration)["token"]
status, _headers, chat, _ = client.call(
"POST",
"/api/v1/chats",
body={"title": "Idempotency race", "member_ids": [alice["id"], bob["id"]]},
token=alice_token,
)
if status != 201:
raise AssertionError(f"chat creation failed: HTTP {status}")
key = uuid.uuid4().hex
text = "concurrent replay " + key
path = "/api/v1/chats/" + chat["id"] + "/messages"
def send_same_body(_index: int):
return client.call(
"POST", path, body={"text": text}, token=alice_token,
headers={"Idempotency-Key": key},
)
with ThreadPoolExecutor(max_workers=args.workers) as pool:
results = list(pool.map(send_same_body, range(args.workers)))
statuses = Counter(status for status, _h, _p, _r in results)
statuses.pop(200, None)
if set(statuses) != {201}:
raise AssertionError(f"concurrent replay statuses are not all 201: {dict(statuses)}")
ids = {payload["id"] for _s, _h, payload, _r in results}
if len(ids) != 1:
raise AssertionError(f"concurrent replay created {len(ids)} messages instead of 1")
instances = {
headers.get("X-Instance-Id", "")
for _s, headers, _p, _r in results
if headers.get("X-Instance-Id")
}
if len(instances) < 2:
raise AssertionError("concurrent replay did not reach at least two replicas")
message_id = next(iter(ids))
status, _headers, page, _ = client.call(
"GET", path + "?limit=100", token=bob_token
)
if status != 200:
raise AssertionError(f"history read failed: HTTP {status}")
matching = [m for m in page["items"] if m["id"] == message_id]
if len(matching) != 1:
raise AssertionError("history contains a duplicate or a missing replayed message")
# Same key, different canonical body -> 409.
status, _headers, payload, _ = client.call(
"POST", path, body={"text": text + " different"}, token=alice_token,
headers={"Idempotency-Key": key},
)
if status != 409 or payload.get("error", {}).get("code") != "conflict":
raise AssertionError(f"different body with the same key returned HTTP {status}, expected 409")
# Missing / malformed key -> 400.
for headers in ({}, {"Idempotency-Key": "bad key with spaces"}):
status, _headers, payload, _ = client.call(
"POST", path, body={"text": "no valid key"}, token=alice_token, headers=headers
)
if status != 400 or payload.get("error", {}).get("code") != "invalid_request":
raise AssertionError(f"invalid Idempotency-Key returned HTTP {status}, expected 400")
# Scope includes the user: the same key by another user is independent.
status, _headers, bob_message, _ = client.call(
"POST", path, body={"text": "bob independent"}, token=bob_token,
headers={"Idempotency-Key": key},
)
if status != 201 or bob_message["sender_id"] != bob["id"]:
raise AssertionError(f"another user's identical key was not independent: HTTP {status}")
# Scope includes the path: the same key in another chat is independent.
status, _headers, other_chat, _ = client.call(
"POST",
"/api/v1/chats",
body={"title": "Second chat", "member_ids": [alice["id"], bob["id"]]},
token=alice_token,
)
if status != 201:
raise AssertionError(f"second chat creation failed: HTTP {status}")
status, _headers, other_message, _ = client.call(
"POST",
"/api/v1/chats/" + other_chat["id"] + "/messages",
body={"text": "same key another chat"},
token=alice_token,
headers={"Idempotency-Key": key},
)
if status != 201 or other_message["id"] == message_id:
raise AssertionError(f"same key in another chat was not independent: HTTP {status}")
print("concurrent_workers:", args.workers)
print("statuses:", dict(statuses))
print("instances:", ", ".join(sorted(instances)))
print("PASS: shared idempotency creates exactly one message and respects all scopes")
if __name__ == "__main__":
main()
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Prove that one session is served by at least two real API replicas via the LB."""
from __future__ import annotations
import argparse
from collections import Counter
from common import HttpClient
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--requests", type=int, default=60)
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_registration = client.register()
bob, _bob_registration = client.register()
alice_session = client.login(alice_registration)
token = alice_session["token"]
status, _headers, chat, _ = client.call(
"POST",
"/api/v1/chats",
body={"title": "Replica spread", "member_ids": [alice["id"], bob["id"]]},
token=token,
)
if status != 201:
raise AssertionError(f"chat creation failed: HTTP {status}")
instances: Counter[str] = Counter()
for _ in range(args.requests):
status, headers, me, _ = client.call("GET", "/api/v1/users/me", token=token)
if status != 200 or me["id"] != alice["id"]:
raise AssertionError("existing session did not work through the LB")
instance = headers.get("X-Instance-Id", "")
if not instance:
raise AssertionError("missing X-Instance-Id on an API response")
if not headers.get("X-Request-Id"):
raise AssertionError("missing X-Request-Id on an API response")
instances[instance] += 1
if len(instances) < 2:
raise AssertionError(
"fewer than two API instances observed; check LB discovery / replica count"
)
# A write accepted by one replica must be visible from any other replica.
for index, instance in enumerate(instances):
status, headers, message, _ = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/messages",
body={"text": f"replica-write-{index}"},
token=token,
headers={"Idempotency-Key": f"spread-{index}-{alice['id']}"},
)
if status != 201:
raise AssertionError(f"message write failed through {instance}: HTTP {status}")
status, _headers, page, _ = client.call(
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", token=token
)
if status != 200 or not any(m["id"] == message["id"] for m in page["items"]):
raise AssertionError("message written by one replica is not readable")
print("instances:", ", ".join(f"{name}={count}" for name, count in sorted(instances.items())))
print("PASS: one session served by >=2 API replicas with a shared datastore")
if __name__ == "__main__":
main()
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
"""Print the effective PostgreSQL timeouts and pool limits.
Run inside the API container:
docker compose exec -T api python student-tests/pool_settings.py
"""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from app import db
def main() -> None:
db.pool().open(wait=True, timeout=5)
db.apply_migrations()
with db.pool().connection() as conn:
statement = conn.execute("SHOW statement_timeout").fetchone()["statement_timeout"]
idle = conn.execute("SHOW idle_in_transaction_session_timeout").fetchone()[
"idle_in_transaction_session_timeout"
]
lock = conn.execute("SHOW lock_timeout").fetchone()["lock_timeout"]
stats = db.pool().get_stats()
print("statement_timeout =", statement)
print("idle_in_transaction_session_timeout =", idle)
print("lock_timeout =", lock)
print("pool_min =", stats["pool_min"])
print("pool_max =", stats["pool_max"])
print("pool_size =", stats["pool_size"])
print("pool_available =", stats["pool_available"])
print("PASS: bounded pool and SQL timeouts are active")
if __name__ == "__main__":
main()
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Lab5 track 1: presigned direct upload plus the mandatory proxy fallback."""
from __future__ import annotations
import argparse
import hashlib
import io
import uuid
from PIL import Image
from common import HttpClient, assert_status, error_code
def png_bytes(width: int = 640, height: int = 400) -> bytes:
buffer = io.BytesIO()
Image.new("RGB", (width, height), (20, 160, 90)).save(buffer, format="PNG")
return buffer.getvalue()
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_reg = client.register()
bob, bob_reg = client.register()
alice_token = client.login(alice_reg)["token"]
bob_token = client.login(bob_reg)["token"]
status, _h, chat, _r = client.call(
"POST",
"/api/v1/chats",
body={"title": "Presigned upload", "member_ids": [alice["id"], bob["id"]]},
token=alice_token,
)
assert_status(status, 201, "create chat")
content = png_bytes()
digest = hashlib.sha256(content).hexdigest()
body = {
"filename": "direct.png",
"content_type": "image/png",
"size_bytes": len(content),
"sha256": digest,
}
status, _h, presign, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body=body,
token=alice_token,
)
assert_status(status, 201, "presign")
if presign["method"] != "PUT" or not presign["upload_url"].startswith("http"):
raise AssertionError("presign response is missing a usable PUT URL")
# The mandatory proxy endpoint still works.
status, _h, proxy, _r = client.raw(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=proxy.png",
data=content,
headers={"Authorization": "Bearer " + alice_token, "Content-Type": "image/png"},
)
assert_status(status, 201, "proxy upload remains available")
# Direct PUT to the private bucket through the short-lived signature.
status, _h, _p, _r = client.raw(
"PUT",
presign["upload_url"],
data=content,
headers=presign["headers"],
)
assert_status(status, 200, "presigned PUT")
status, _h, ready, _r = client.call(
"POST",
"/api/v1/attachments/" + presign["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=alice_token,
)
assert_status(status, 200, "complete")
if ready["status"] != "ready" or ready["sha256"] != digest:
raise AssertionError("completed attachment is not ready with the expected checksum")
# A chat member can read the bytes through the API; no redirect is used.
status, headers, _p, raw = client.raw(
"GET",
"/api/v1/attachments/" + presign["attachment_id"] + "/content",
headers={"Authorization": "Bearer " + bob_token},
)
assert_status(status, 200, "download direct upload")
if raw != content or headers.get_content_type() != "image/png":
raise AssertionError("direct-upload download differs from the original")
# Complete is idempotent for the same parameters.
status, _h, again, _r = client.call(
"POST",
"/api/v1/attachments/" + presign["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=alice_token,
)
assert_status(status, 200, "idempotent complete")
if again["id"] != ready["id"]:
raise AssertionError("idempotent complete returned another attachment")
# Negative: a ready attachment cannot be completed with different values.
for payload, label in (
({"sha256": "0" * 64, "size_bytes": len(content)}, "ready checksum mismatch"),
({"sha256": digest, "size_bytes": len(content) + 1}, "ready size mismatch"),
):
status, _h, error, _r = client.call(
"POST",
"/api/v1/attachments/" + presign["attachment_id"] + "/complete",
body=payload,
token=alice_token,
)
assert_status(status, 409, label)
if error_code(error) == "internal_error":
raise AssertionError(label + " leaked a 500")
# Negative: wrong bytes that are never promoted (checksum/size mismatch).
for wrong, label in (
(png_bytes(641, 400), "checksum mismatch"),
(content + b"x", "size mismatch"),
):
status, _h, bad_upload, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body=body,
token=alice_token,
)
assert_status(status, 201, label + " presign")
status, _h, _p, _r = client.raw(
"PUT",
bad_upload["upload_url"],
data=wrong,
headers=bad_upload["headers"],
)
assert_status(status, 200, label + " PUT")
status, _h, error, _r = client.call(
"POST",
"/api/v1/attachments/" + bad_upload["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=alice_token,
)
assert_status(status, 400, label)
if error_code(error) == "internal_error":
raise AssertionError(label + " leaked a 500")
# Negative: another user cannot complete Alice's pending upload.
status, _h, pending, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body=body,
token=alice_token,
)
assert_status(status, 201, "second presign")
status, _h, error, _r = client.call(
"POST",
"/api/v1/attachments/" + pending["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=bob_token,
)
assert_status(status, 404, "foreign complete")
if error_code(error) == "internal_error":
raise AssertionError("foreign complete leaked a 500")
# Negative: completing before PUT is a typed 409.
status, _h, error, _r = client.call(
"POST",
"/api/v1/attachments/" + pending["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=alice_token,
)
assert_status(status, 409, "incomplete direct upload")
if error_code(error) == "internal_error":
raise AssertionError("incomplete upload leaked a 500")
# Negative: bad presign input.
for bad, expected in (
({**body, "content_type": "image/gif"}, 415),
({**body, "size_bytes": 10_485_761}, 400),
({**body, "sha256": "ABC"}, 400),
({**body, "filename": "../x.png"}, 400),
):
status, _h, error, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body=bad,
token=alice_token,
)
assert_status(status, expected, "bad presign input")
if error_code(error) == "internal_error":
raise AssertionError("bad presign input leaked a 500")
print(
"PASS: presigned upload, complete, checksum verification, proxy fallback, "
"idempotency and negative cases"
)
if __name__ == "__main__":
main()
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
"""Lab4 infrastructure checks: private bucket and no published admin UIs."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import urllib.error
import urllib.request
import uuid
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import ClientError
ROOT = Path(__file__).resolve().parents[1]
def read_env() -> dict[str, str]:
values = dict(os.environ)
path = ROOT / ".env"
if path.is_file():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
values.setdefault(key.strip(), value.strip())
return values
def s3_client(env: dict[str, str]):
return boto3.client(
"s3",
endpoint_url=env["S3_PUBLIC_ENDPOINT"],
region_name=env.get("S3_REGION", "us-east-1"),
aws_access_key_id=env["S3_ACCESS_KEY_ID"],
aws_secret_access_key=env["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
def check_public_ports() -> None:
result = subprocess.run(
["docker", "compose", "--env-file", ".env", "ps", "--format", "json"],
cwd=ROOT,
capture_output=True,
text=True,
timeout=30,
)
if result.returncode != 0:
raise AssertionError("could not inspect compose publishers")
services = [json.loads(line) for line in result.stdout.splitlines() if line.strip()]
for service in services:
name = service.get("Service") or service.get("Name", "")
publishers = service.get("Publishers") or []
published = [p for p in publishers if p.get("PublishedPort")]
if name == "seaweedfs":
ports = {int(p["PublishedPort"]) for p in published}
if ports != {int(os.environ.get("S3_PORT", read_env().get("S3_PORT", "9000")))}:
raise AssertionError(f"unexpected published SeaweedFS ports: {ports}")
elif name in ("api", "postgres", "redis", "migrate", "s3-init"):
if published:
raise AssertionError(f"{name} publishes a host port: {published}")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
del args
env = read_env()
for key in ("S3_PUBLIC_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"):
if not env.get(key):
raise AssertionError(f"missing {key} in environment/.env")
client = s3_client(env)
key = "reconcile-tests/private-bucket-" + uuid.uuid4().hex
client.put_object(Bucket=env["S3_BUCKET"], Key=key, Body=b"private", ContentType="text/plain")
try:
url = env["S3_PUBLIC_ENDPOINT"].rstrip("/") + "/" + env["S3_BUCKET"] + "/" + key
try:
urllib.request.urlopen(url, timeout=10)
except urllib.error.HTTPError as error:
if error.code not in (401, 403):
raise AssertionError(f"anonymous GET returned HTTP {error.code}, expected 403") from error
else:
raise AssertionError("anonymous GET succeeded on a private bucket")
for operation in ("create_bucket", "delete_bucket"):
try:
if operation == "create_bucket":
client.create_bucket(Bucket="forbidden-" + uuid.uuid4().hex[:12])
else:
client.delete_bucket(Bucket=env["S3_BUCKET"])
except ClientError as error:
if "AccessDenied" not in str(error):
raise AssertionError(f"{operation} failed with an unexpected error: {error}") from error
else:
raise AssertionError(f"restricted identity was allowed to {operation}")
finally:
client.delete_object(Bucket=env["S3_BUCKET"], Key=key)
check_public_ports()
print("PASS: bucket rejects anonymous reads; API identity cannot administer it; no admin UI is published")
if __name__ == "__main__":
main()
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env python3
"""Concurrent registration with the same username: one 201, the rest 409."""
from __future__ import annotations
import argparse
import uuid
from concurrent.futures import ThreadPoolExecutor
from common import HttpClient, assert_status, error_code
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--workers", type=int, default=8)
args = parser.parse_args()
client = HttpClient(args.base_url)
username = "race_" + uuid.uuid4().hex[:16]
body = {
"username": username,
"display_name": "Race candidate",
"password": "T3st-" + uuid.uuid4().hex,
}
def register(_index: int):
status, _, payload, _ = client.call("POST", "/api/v1/users", body)
return status, error_code(payload)
with ThreadPoolExecutor(max_workers=args.workers) as executor:
results = list(executor.map(register, range(args.workers)))
statuses = [status for status, _code in results]
if statuses.count(201) != 1 or statuses.count(409) != args.workers - 1:
raise AssertionError(f"expected one 201 and {args.workers - 1} conflicts, got {statuses}")
for status, code in results:
if status == 409 and code != "conflict":
raise AssertionError("409 must carry conflict code")
session = client.login(body)
assert session["user"]["username"] == username
print(f"PASS: {args.workers} concurrent registrations produced exactly one user and one session")
if __name__ == "__main__":
main()
+166
View File
@@ -0,0 +1,166 @@
#!/usr/bin/env python3
"""Lab4/Lab5 evidence: safe S3/PostgreSQL reconciliation with a grace period."""
from __future__ import annotations
import argparse
import json
import os
import subprocess
import uuid
from pathlib import Path
import boto3
from botocore.config import Config
from common import HttpClient, assert_status
ROOT = Path(__file__).resolve().parents[1]
def read_env() -> dict[str, str]:
values = dict(os.environ)
path = ROOT / ".env"
if path.is_file():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if not line or line.startswith("#") or "=" not in line:
continue
key, _, value = line.partition("=")
values.setdefault(key.strip(), value.strip())
return values
def s3_client(env: dict[str, str]):
return boto3.client(
"s3",
endpoint_url=env["S3_PUBLIC_ENDPOINT"],
region_name=env.get("S3_REGION", "us-east-1"),
aws_access_key_id=env["S3_ACCESS_KEY_ID"],
aws_secret_access_key=env["S3_SECRET_ACCESS_KEY"],
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}),
)
def reconcile(extra: list[str]) -> dict:
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"--profile",
"tools",
"run",
"--rm",
"--no-deps",
"reconcile",
"python",
"-m",
"app.reconcile",
*extra,
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=120,
)
if result.returncode != 0:
raise AssertionError(
"reconciler failed: " + (result.stderr or result.stdout)[-500:]
)
for line in reversed(result.stdout.splitlines()):
line = line.strip()
if line.startswith("{"):
return json.loads(line)
raise AssertionError("reconciler did not print a JSON summary")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
args = parser.parse_args()
env = read_env()
client = s3_client(env)
bucket = env["S3_BUCKET"]
# 1. A fresh orphan is reported by a dry-run and is not deleted.
orphan_key = "reconcile-tests/orphan-" + uuid.uuid4().hex
client.put_object(Bucket=bucket, Key=orphan_key, Body=b"orphan", ContentType="text/plain")
dry = reconcile(
["--dry-run", "--orphan-grace-seconds", "0", "--pending-grace-seconds", "0"]
)
if dry["orphans_stale_deleted"] < 1:
raise AssertionError("dry-run did not report the injected orphan")
client.head_object(Bucket=bucket, Key=orphan_key)
# 2. A fresh orphan inside the default grace window is kept by --apply.
kept_key = "reconcile-tests/kept-" + uuid.uuid4().hex
client.put_object(Bucket=bucket, Key=kept_key, Body=b"kept", ContentType="text/plain")
kept = reconcile(
["--apply", "--orphan-grace-seconds", "86400", "--pending-grace-seconds", "86400"]
)
if kept["orphans_fresh_kept"] < 1:
raise AssertionError("grace-window orphan was not reported as kept")
client.head_object(Bucket=bucket, Key=kept_key)
# 3. --apply with grace 0 removes both injected objects.
applied = reconcile(
["--apply", "--orphan-grace-seconds", "0", "--pending-grace-seconds", "0"]
)
if applied["orphans_stale_deleted"] < 2:
raise AssertionError("apply did not delete the injected orphans")
for key in (orphan_key, kept_key):
try:
client.head_object(Bucket=bucket, Key=key)
except Exception:
pass
else:
raise AssertionError("orphan still exists after apply")
# 4. An interrupted presigned upload is removed with its metadata row.
http = HttpClient(args.base_url)
alice, alice_reg = http.register()
token = http.login(alice_reg)["token"]
status, _h, chat, _r = http.call(
"POST",
"/api/v1/chats",
body={"title": "Reconcile pending", "member_ids": [alice["id"]]},
token=token,
)
assert_status(status, 201, "create chat")
import hashlib
import io
from PIL import Image
buffer = io.BytesIO()
Image.new("RGB", (32, 32), (10, 20, 30)).save(buffer, format="PNG")
png = buffer.getvalue()
digest = hashlib.sha256(png).hexdigest()
status, _h, pending, _r = http.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body={
"filename": "pending.png",
"content_type": "image/png",
"size_bytes": len(png),
"sha256": digest,
},
token=token,
)
assert_status(status, 201, "presign pending")
pending_summary = reconcile(
["--apply", "--orphan-grace-seconds", "86400", "--pending-grace-seconds", "0"]
)
if pending_summary["stale_pending_deleted"] < 1:
raise AssertionError("stale pending upload was not removed")
status, _h, _p, _r = http.call(
"GET", "/api/v1/attachments/" + pending["attachment_id"], token=token
)
assert_status(status, 404, "pending metadata removed")
print("PASS: dry-run/apply reconciliation, grace period, orphan cleanup, stale pending cleanup")
if __name__ == "__main__":
main()
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Prove that a failure after the chat insert rolls back the whole chat.
Run inside the API container (psycopg is installed there):
docker compose exec -T api python student-tests/rollback_chat.py
A temporary trigger aborts the second chat_members insert. The API must
return a bounded 500 and leave no chat row, because chat + membership share
one transaction.
"""
from __future__ import annotations
import argparse
import os
import uuid
import psycopg
from common import HttpClient, assert_status, error_code
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL"))
args = parser.parse_args()
if not args.database_url:
raise SystemExit("DATABASE_URL is required")
client = HttpClient(args.base_url)
actor, actor_registration = client.register()
victim, _victim_registration = client.register()
session = client.login(actor_registration)
token = session["token"]
title = "rollback-" + uuid.uuid4().hex
function_name = "test_fail_member_" + uuid.uuid4().hex[:8]
victim_id = str(uuid.UUID(victim["id"]))
with psycopg.connect(args.database_url, autocommit=True) as conn:
conn.execute(
f"""
CREATE FUNCTION {function_name}() RETURNS trigger AS $$
BEGIN
IF NEW.user_id = '{victim_id}'::uuid THEN
RAISE EXCEPTION 'injected membership failure';
END IF;
RETURN NEW;
END
$$ LANGUAGE plpgsql
"""
)
conn.execute(
f"CREATE TRIGGER {function_name} BEFORE INSERT ON chat_members "
f"FOR EACH ROW EXECUTE FUNCTION {function_name}()"
)
try:
status, _, payload, _ = client.call(
"POST",
"/api/v1/chats",
{"title": title, "member_ids": [actor["id"], victim["id"]]},
token=token,
)
assert_status(status, 500, "injected membership failure")
if error_code(payload) != "internal_error":
raise AssertionError("unexpected error code for injected failure")
rows = conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone()
if rows[0] != 0:
raise AssertionError("chat survived a failed membership transaction")
finally:
conn.execute(f"DROP TRIGGER IF EXISTS {function_name} ON chat_members")
conn.execute(f"DROP FUNCTION IF EXISTS {function_name}()")
status, _, chat, _ = client.call(
"POST",
"/api/v1/chats",
{"title": title + "-ok", "member_ids": [actor["id"], victim["id"]]},
token=token,
)
assert_status(status, 201, "chat after trigger cleanup")
assert set(chat["member_ids"]) == {actor["id"], victim["id"]}
print("PASS: failed membership insert rolled back the chat; subsequent create is atomic")
if __name__ == "__main__":
main()
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
"""Lab5 track 1: compare API traffic/RAM for proxy vs presigned direct upload."""
from __future__ import annotations
import argparse
import hashlib
import io
import json
import math
import os
import re
import subprocess
import threading
import time
from PIL import Image
from common import HttpClient, assert_status
MEM_RE = re.compile(r"([0-9.]+)\s*(B|KiB|MiB|GiB)")
def make_png(target_bytes: int) -> bytes:
pixels = max(1, target_bytes // 3)
width = max(32, int(math.sqrt(pixels * 1.4)))
height = max(32, pixels // width)
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
buffer = io.BytesIO()
image.save(buffer, format="PNG", compress_level=1)
return buffer.getvalue()
def api_containers(prefix: str) -> list[str]:
try:
result = subprocess.run(
["docker", "ps", "--filter", "name=" + prefix, "--format", "{{.Names}}"],
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.SubprocessError):
return []
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
def api_net_bytes(names: list[str]) -> dict[str, int]:
total = 0
for name in names:
try:
result = subprocess.run(
["docker", "exec", name, "cat", "/proc/net/dev"],
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.SubprocessError):
continue
for line in result.stdout.splitlines():
if "eth0:" not in line:
continue
fields = line.split("eth0:")[1].split()
# RX bytes is field 0; TX bytes is field 8.
total += int(fields[0]) + int(fields[8])
return {"total": total}
def memory_mib(text: str) -> float | None:
match = MEM_RE.search(text)
if not match:
return None
return float(match.group(1)) * {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[match.group(2)]
def api_mem_mib(prefix: str) -> float:
try:
result = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.SubprocessError):
return 0.0
total = 0.0
for line in result.stdout.splitlines():
name, _, mem = line.partition(" ")
if prefix in name:
value = memory_mib(mem)
if value is not None:
total += value
return total
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--container-prefix", default="mtusi-lab4-api")
parser.add_argument("--output", default="")
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_reg = client.register()
token = client.login(alice_reg)["token"]
status, _h, chat, _r = client.call(
"POST",
"/api/v1/chats",
body={"title": "Transfer comparison", "member_ids": [alice["id"]]},
token=token,
)
assert_status(status, 201, "create chat")
names = api_containers(args.container_prefix)
results = []
for target in (512 * 1024, 4 * 1024 * 1024):
content = make_png(target)
digest = hashlib.sha256(content).hexdigest()
# Proxy: client -> API -> S3, all bytes pass through the API process.
before = api_net_bytes(names)["total"]
mem_before = api_mem_mib(args.container_prefix)
status, _h, _p, _r = client.raw(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=proxy.png",
data=content,
headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"},
timeout=120,
)
assert_status(status, 201, "proxy upload")
after = api_net_bytes(names)["total"]
mem_after = api_mem_mib(args.container_prefix)
proxy = {
"file_bytes": len(content),
"api_network_bytes": after - before,
"api_memory_growth_mib": round(max(0.0, mem_after - mem_before), 2),
"client_to_api_bytes": len(content),
"client_to_s3_bytes": 0,
}
# Direct: presign, client -> S3, then complete reads the object once to
# verify the checksum. The public API never receives the file body.
status, _h, presign, _r = client.call(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments/presign",
body={
"filename": "direct.png",
"content_type": "image/png",
"size_bytes": len(content),
"sha256": digest,
},
token=token,
)
assert_status(status, 201, "presign")
status, _h, _p, _r = client.raw(
"PUT", presign["upload_url"], data=content, headers=presign["headers"], timeout=120
)
assert_status(status, 200, "direct PUT")
before = api_net_bytes(names)["total"]
mem_before = api_mem_mib(args.container_prefix)
peak = {"value": mem_before}
stop = threading.Event()
def sampler() -> None:
while not stop.is_set():
peak["value"] = max(peak["value"], api_mem_mib(args.container_prefix))
time.sleep(0.1)
thread = threading.Thread(target=sampler, daemon=True)
thread.start()
status, _h, _p, _r = client.call(
"POST",
"/api/v1/attachments/" + presign["attachment_id"] + "/complete",
body={"sha256": digest, "size_bytes": len(content)},
token=token,
)
stop.set()
thread.join(timeout=2)
assert_status(status, 200, "complete")
after = api_net_bytes(names)["total"]
direct = {
"file_bytes": len(content),
"api_network_bytes": after - before,
"api_memory_peak_mib": round(peak["value"], 2),
"api_memory_growth_mib": round(max(0.0, peak["value"] - mem_before), 2),
"client_to_api_bytes": 0,
"client_to_s3_bytes": len(content),
}
results.append({"proxy": proxy, "direct": direct})
summary = {"container_prefix": args.container_prefix, "sizes": results}
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
if args.output:
with open(args.output, "w", encoding="utf-8") as handle:
json.dump(summary, handle, ensure_ascii=False, indent=2, sort_keys=True)
print("PASS: measured proxy vs direct API traffic and memory at two file sizes")
if __name__ == "__main__":
main()
+147
View File
@@ -0,0 +1,147 @@
#!/usr/bin/env python3
"""Lab4 experiment: 10+ concurrent uploads with API container peak memory.
The uploads are streamed to an ephemeral file and decoded under a bounded
semaphore, so this test looks for a flat memory profile rather than a linear
growth with concurrency.
"""
from __future__ import annotations
import argparse
import io
import json
import os
import re
import subprocess
import threading
import time
from concurrent.futures import ThreadPoolExecutor
from PIL import Image
from common import HttpClient, assert_status
MEM_RE = re.compile(r"([0-9.]+)\s*(B|KiB|MiB|GiB)")
def make_png(width: int, height: int) -> bytes:
# Random pixels compress poorly, giving a realistic multi-MiB upload.
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
buffer = io.BytesIO()
image.save(buffer, format="PNG", compress_level=1)
return buffer.getvalue()
def memory_mib(text: str) -> float | None:
match = MEM_RE.search(text)
if not match:
return None
value = float(match.group(1))
unit = match.group(2)
return value * {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
def sample_api_memory(prefix: str) -> dict[str, float]:
try:
result = subprocess.run(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{.Name}} {{.MemUsage}}",
],
capture_output=True,
text=True,
timeout=10,
)
except (OSError, subprocess.SubprocessError):
return {}
if result.returncode != 0:
return {}
usage = {}
for line in result.stdout.splitlines():
name, _, mem = line.partition(" ")
if prefix in name:
value = memory_mib(mem)
if value is not None:
usage[name] = value
return usage
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--workers", type=int, default=10)
parser.add_argument("--width", type=int, default=1400)
parser.add_argument("--height", type=int, default=1000)
parser.add_argument("--container-prefix", default="mtusi-lab4-api")
parser.add_argument("--output", default="")
args = parser.parse_args()
client = HttpClient(args.base_url)
alice, alice_reg = client.register()
alice_token = client.login(alice_reg)["token"]
status, _h, chat, _r = client.call(
"POST",
"/api/v1/chats",
body={"title": "Upload memory experiment", "member_ids": [alice["id"]]},
token=alice_token,
)
assert_status(status, 201, "create chat")
content = make_png(args.width, args.height)
peak: dict[str, float] = {}
stop = threading.Event()
def sampler() -> None:
while not stop.is_set():
for name, value in sample_api_memory(args.container_prefix).items():
peak[name] = max(peak.get(name, 0.0), value)
time.sleep(0.2)
sampler_thread = threading.Thread(target=sampler, name="memory-sampler", daemon=True)
sampler_thread.start()
baseline = sample_api_memory(args.container_prefix)
def upload(index: int):
return client.raw(
"POST",
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-%d.png" % index,
data=content,
headers={"Authorization": "Bearer " + alice_token, "Content-Type": "image/png"},
timeout=120,
)
started = time.monotonic()
with ThreadPoolExecutor(max_workers=args.workers) as pool:
results = list(pool.map(upload, range(args.workers)))
elapsed = time.monotonic() - started
stop.set()
sampler_thread.join(timeout=2)
statuses = [status for status, _h, _p, _r in results]
assert_status(min(statuses), 201, "concurrent uploads")
if max(statuses) != 201:
raise AssertionError(f"some uploads failed: {statuses}")
summary = {
"workers": args.workers,
"file_bytes": len(content),
"elapsed_seconds": round(elapsed, 3),
"successful_uploads": len(statuses),
"baseline_mib": {k: round(v, 2) for k, v in baseline.items()},
"peak_mib": {k: round(v, 2) for k, v in peak.items()},
"peak_growth_mib": {
k: round(peak.get(k, 0.0) - baseline.get(k, 0.0), 2) for k in peak
},
}
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
if args.output:
with open(args.output, "w", encoding="utf-8") as handle:
json.dump(summary, handle, ensure_ascii=False, indent=2, sort_keys=True)
print("PASS: concurrent uploads completed; inspect peak_mib for the bounded-memory claim")
if __name__ == "__main__":
main()