forked from mtusi2026/mtusi2026lab4
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
935307d412 |
@@ -17,4 +17,6 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
tests
|
||||
*.log
|
||||
.DS_Store
|
||||
|
||||
+26
-10
@@ -1,20 +1,36 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# The values below are placeholders, not production secrets.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
# Host port for the load balancer; the test BASE_URL must match it.
|
||||
LAB4_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
API_REPLICAS=2
|
||||
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-local-postgres-password
|
||||
POSTGRES_DB=messenger
|
||||
|
||||
# SeaweedFS administrative identity is used only by the one-shot bootstrap.
|
||||
S3_ADMIN_ACCESS_KEY_ID=seaweedfs-admin
|
||||
S3_ADMIN_SECRET_ACCESS_KEY=change-me-local-seaweedfs-admin-password
|
||||
|
||||
# Separate S3 identity used by the API. Its SeaweedFS actions are Read/Write/List
|
||||
# only; bucket creation and administration use the bootstrap identity.
|
||||
S3_ENDPOINT=http://seaweedfs:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=change-me-local-api-secret
|
||||
|
||||
# Must be identical for all API replicas so pagination cursors work across the LB.
|
||||
CURSOR_SECRET=change-me-local-cursor-secret
|
||||
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
|
||||
# The application also accepts *_FILE variants (for example DATABASE_URL_FILE
|
||||
# and S3_SECRET_ACCESS_KEY_FILE) when running outside Compose.
|
||||
# DATABASE_URL=postgresql://messenger:change-me-local-postgres-password@localhost:5432/messenger
|
||||
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
FROM python:3.12.7-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN groupadd --system messenger \
|
||||
&& useradd --system --gid messenger --home-dir /app --shell /usr/sbin/nologin messenger
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
|
||||
USER messenger
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["python", "-m", "app.main"]
|
||||
@@ -4,10 +4,11 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover student-tests test-negative
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "student-tests: unit tests of validation helpers; test-negative: attachment rejection scenarios against a running stand"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -26,5 +27,11 @@ test-persistence:
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
student-tests:
|
||||
$(PYTHON) -m unittest discover -s student-tests -v
|
||||
|
||||
test-negative:
|
||||
$(PYTHON) student-tests/test_api_negative.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,37 +1,192 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Ким Владимир, БВТ2302
|
||||
- Целевая оценка; 3
|
||||
- Commit/PR: ветка `solution/bvt2302-kim`; PR в исходный репозиторий открывается после push.
|
||||
- Использованные источники, библиотеки и помощь ИИ: контракт и рубрика репозитория; документация S3
|
||||
(upload/presigned), SeaweedFS (S3 gateway, `s3.json` identities), boto3/botocore, psycopg 3, Pillow,
|
||||
HAProxy.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Версии закреплены:
|
||||
|
||||
| Роль | Образ | Версия |
|
||||
| --- | --- | --- |
|
||||
| API | `python:3.12.7-slim-bookworm` (digest `sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d`) + `requirements.txt` | boto3 1.35.36, Pillow 11.0.0, psycopg 3.2.3, psycopg_pool 3.2.3 |
|
||||
| PostgreSQL | `postgres:16.4-alpine` | 16.4 |
|
||||
| S3 | `chrislusf/seaweedfs:3.71` | 3.71 |
|
||||
| Load balancer | `haproxy:3.0.8-alpine` | 3.0.8 |
|
||||
|
||||
```sh
|
||||
# 1. Локальная конфигурация: секреты только в .env, файл в Git не попадает.
|
||||
cp .env.example .env
|
||||
# Отредактировать пароли/секреты. Для локального стенда достаточно синтетических значений.
|
||||
|
||||
# 2. Проверка структуры и конфигурации Compose.
|
||||
python scripts/check_contract.py
|
||||
python scripts/validate_openapi.py # требует requirements-dev.txt
|
||||
# На Windows глобальный Python может не иметь openapi-spec-validator;
|
||||
# полная валидация выполнялась интерпретатором из .venv:
|
||||
# .\.venv\Scripts\python.exe scripts/validate_openapi.py
|
||||
docker compose --env-file .env config --quiet
|
||||
|
||||
# 3. Сборка и запуск. Bucket создаёт one-shot контейнер `bootstrap`; схему
|
||||
# PostgreSQL мигрирует каждая API-реплика при старте (идемпотентно, под
|
||||
# advisory lock).
|
||||
docker compose --env-file .env up -d --build
|
||||
|
||||
# 4. Готовность (PostgreSQL + S3). LAB4_PORT по умолчанию 8080;
|
||||
# на машине автора порт 8080 занят стендом lab3, поэтому в .env стоит 8081.
|
||||
curl -fsS http://127.0.0.1:8081/health/ready
|
||||
|
||||
# 5. Публичный smoke и собственные проверки.
|
||||
python tests/smoke.py --base-url http://127.0.0.1:8081
|
||||
python -m unittest discover -s student-tests -v
|
||||
python student-tests/test_api_negative.py --base-url http://127.0.0.1:8081
|
||||
|
||||
# 6. Сценарии отказов (явные, останавливают инфраструктуру).
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8081
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8081
|
||||
python scripts/evidence_s3_outage.py
|
||||
```
|
||||
|
||||
`scripts/restart.sh` / `scripts/stop-one.sh` — POSIX-варианты для Linux/macOS (`chmod +x` нужен перед
|
||||
запуском). `scripts/restart.cmd` / `scripts/stop-one.cmd` — эквиваленты для Windows: тест запускает hook
|
||||
без shell interpolation, поэтому на Windows нужен исполняемый файл с расширением `.cmd`.
|
||||
|
||||
Безопасная остановка (данные сохраняются):
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env stop
|
||||
```
|
||||
|
||||
Намеренное удаление учебных данных (необратимо, только для локального стенда):
|
||||
|
||||
```sh
|
||||
docker compose --env-file .env down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ──HTTP──> HAProxy :8080 ──> api1:8080 ─┐
|
||||
│ ├──> PostgreSQL (users, chats, members,
|
||||
└────────> api2:8080 ────┘ messages, attachments, idempotency)
|
||||
└──> SeaweedFS S3 :8333 (приватный bucket,
|
||||
только байты изображений)
|
||||
```
|
||||
|
||||
- **Источник истины** — PostgreSQL. Метаданные вложения (`id`, `chat_id`, `owner_id`, `filename`,
|
||||
`content_type`, `size_bytes`, `sha256`, `object_key`, `status`, `variants`, `error_code`, таймстемпы)
|
||||
хранятся только там. Байты — только в приватном bucket; локального каталога с оригиналами у API нет.
|
||||
- **Object key** назначает сервер: `chats/<chat_id>/<attachment_id>/original`. `filename` — только
|
||||
отображаемое имя, в путь/ключ не попадает.
|
||||
- **Транзакционные границы.** Upload: проверка доступа и лимитов → проверка изображения → `PUT` в S3 →
|
||||
`INSERT` метаданных в PostgreSQL. Если `INSERT` не удался, объект удаляется компенсацией
|
||||
(`app/server.py`, `_upload_attachment`), поэтому неучтённых объектов не остаётся. Обратный случай
|
||||
(метаданные без объекта) виден как `404` на download и не превращается в 500.
|
||||
- **201 ready** возвращается только после успешной записи объекта и метаданных, то есть когда оригинал
|
||||
действительно доступен любой реплике.
|
||||
- **Идемпотентность сообщений.** Таблица `idempotency(user_id, chat_id, key)` + транзакционный
|
||||
advisory lock по SHA-256 области уникальности; сообщение и запись результата создаются в одной
|
||||
транзакции, повтор с другим body — `409`.
|
||||
- **Сессии.** Непрозрачный токен (32 случайных байта, base64url), в БД только SHA-256 токена и expiry.
|
||||
TTL — `SESSION_TTL_SECONDS` (по умолчанию 3600).
|
||||
- **Пароли** — `scrypt` (N=2^14, r=8, p=1) с солью, сравнение `hmac.compare_digest`; для неизвестного
|
||||
username выполняется фиктивная проверка, чтобы не выдавать существование пользователя по времени.
|
||||
- **Курсоры пагинации** — HMAC-подписанный base64url JSON с `(created_at, id)` и привязкой к ресурсу
|
||||
(`kind`, `chat_id`); чужой/испорченный курсор — `400`.
|
||||
- **Readiness.** `/health/ready` проверяет PostgreSQL и S3, потому что обязательный синхронный путь
|
||||
включает скачивание оригинала. При недоступности S3 реплики исключаются из LB, клиент получает `503`;
|
||||
это осознанный выбор, зафиксированный в `app/server.py`.
|
||||
- **Лимиты.** `Content-Length`/chunked чтение ограничены `MAX_UPLOAD_BYTES` (10 MiB) до чтения тела;
|
||||
MIME проверяется по магическим байтам, число пикселей — по заголовку изображения до полного
|
||||
декодирования (`MAX_IMAGE_PIXELS` = 40 млн). Полное тело ограничено 10 MiB, поэтому RAM одной
|
||||
загрузки ограничена сверху; download идёт чанками по 64 KiB.
|
||||
- **Инфраструктура.** SeaweedFS запускается одним процессом `weed server` с master/volume/filer/S3.
|
||||
Административная идентичность (`S3_ADMIN_*`) используется только one-shot `bootstrap`; API использует
|
||||
отдельную идентичность с действиями `Read`/`Write`/`List`/`Tagging`. Bucket создаётся идемпотентно
|
||||
(`app/bootstrap_s3.py`: `head_bucket` → `create_bucket`, повторный запуск безопасен). Порт S3 и
|
||||
внутренние порты API не публикуются на хост; наружу открыт только LB (loopback).
|
||||
- **LB.** HAProxy в режиме HTTP с `option http-server-close`: без него клиентские keep-alive соединения
|
||||
закреплялись за одной репликой и roundrobin не наблюдался. Имена реплик разрешаются через DNS Docker
|
||||
(`resolvers docker`, `init-addr none`) с периодическим переразрешением: иначе HAProxy запоминал IP,
|
||||
полученный при одновременном старте, и после пересоздания контейнера мог ходить не туда.
|
||||
`timeout connect 500ms`, `retries 2`, `option redispatch` и частые health-check дают переключение
|
||||
на живую реплику за единицы секунд.
|
||||
|
||||
Соответствие переменных `.env.example` реальной конфигурации:
|
||||
|
||||
| Переменная | Где используется |
|
||||
| --- | --- |
|
||||
| `APP_HOST`, `APP_PORT` | адрес прослушивания API (`app/config.py`) |
|
||||
| `DATABASE_URL` | строка подключения psycopg (в Compose собирается из `POSTGRES_*`) |
|
||||
| `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` | сервис PostgreSQL и сборка `DATABASE_URL` |
|
||||
| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET` | клиент S3 и bootstrap |
|
||||
| `S3_ACCESS_KEY_ID`, `S3_SECRET_ACCESS_KEY` | идентичность API в `s3.json` SeaweedFS |
|
||||
| `S3_ADMIN_ACCESS_KEY_ID`, `S3_ADMIN_SECRET_ACCESS_KEY` | идентичность one-shot bootstrap |
|
||||
| `SESSION_TTL_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` | TTL сессий и окно идемпотентности |
|
||||
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | лимиты загрузки |
|
||||
| `CURSOR_SECRET` | HMAC-подпись курсоров, одинаковый у всех реплик |
|
||||
| `LAB4_PORT` | host-порт LB (по умолчанию 8080, у автора 8081) |
|
||||
|
||||
Приложение также поддерживает `*_FILE`-варианты секретов (`DATABASE_URL_FILE`,
|
||||
`S3_SECRET_ACCESS_KEY_FILE` и т.д.) на случай запуска вне Compose.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Офлайн-структура контракта | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | `evidence/check-contract.txt` |
|
||||
| Полная валидация OpenAPI 3.1 | `python scripts/validate_openapi.py` (после `requirements-dev.txt`) | `PASS: full OpenAPI 3.1 and JSON Schema validation` | `evidence/validate-openapi.txt` |
|
||||
| Compose-конфигурация | `docker compose --env-file .env config --quiet` | exit 0, ошибок нет | вывод команды в логе работы |
|
||||
| Публичный smoke (health, users, sessions, chats, messages, pagination, idempotency, upload/download, SHA-256, чужой доступ, logout) | `python tests/smoke.py --base-url http://127.0.0.1:8081` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` |
|
||||
| S3-сервер в общем стенде, приватный bucket | `docker compose ps`; `python scripts/verify_private_bucket.py` (внутри сети) | объекты в bucket есть; анонимный GET объекта — `403`, анонимный list — `403` | `evidence/docker-ps.txt`, `evidence/private-bucket.txt` |
|
||||
| Метаданные в PostgreSQL, байты не в БД/локальном каталоге | `\d attachments`; поиск файлов в контейнере API | таблица `attachments` без BLOB-колонок; в `/app` только код, оригиналов нет | `evidence/db-schema.txt` |
|
||||
| 201 ready после сохранения; побайтное скачивание PNG и JPEG | smoke + `student-tests/test_api_negative.py` | `status=ready`, SHA-256 и размер совпадают, скачанные байты идентичны; отдельный сценарий реального JPEG 640×480 проходит загрузку и скачивание | `evidence/smoke.txt`, `evidence/negative.txt` |
|
||||
| Message с `attachment_ids` читается участниками чата | `student-tests/test_api_negative.py` (сценарий `attachment_ids survive message history`) | элемент истории совпадает с ответом POST, `attachment_ids` сохранены | `evidence/negative.txt` |
|
||||
| PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому; опасные filename; лимиты до декодирования | `python student-tests/test_api_negative.py --base-url ...` | 12/12 сценариев PASS: 413 на размер и пиксели, 415 на битый файл/подмену MIME, 400 на `/`, `\`, NUL, LF | `evidence/negative.txt` |
|
||||
| Чужой чат и чужое вложение | smoke (`eve` → 404), `test_api_negative.py` | `404` на метаданные и bytes, `404` при привязке вложения другого чата, сообщение не создаётся | `evidence/smoke.txt`, `evidence/negative.txt` |
|
||||
| Upload → остановка обслужившей реплики → download через другую | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | 2 реплики до отказа, 112 успешных записей, 0 неопределённых ошибок, восстановление 2.33 с, остановленная реплика не обслуживает, вложение скачано через другую | `evidence/failover.txt` |
|
||||
| Restart стенда с сохранением volumes | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | сессия и данные живы, оригинал скачан побайтно после рестарта всех сервисов | `evidence/persistence.txt` |
|
||||
| Недоступность S3 | `python scripts/evidence_s3_outage.py` | readiness `503`, upload/download `503` (в т.ч. JSON `unavailable` с `X-Request-Id` от реплики), после старта S3 оригинал идентичен | `evidence/s3-outage.txt` |
|
||||
| Unit-проверки валидаторов | `python -m unittest discover -s student-tests -v` | 6 тестов OK (пароли, курсоры, изображения, filename) | `evidence/unit-tests.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Дополнительно: `make test BASE_URL=http://127.0.0.1:8081`, `make test-negative`, `make student-tests` —
|
||||
те же сценарии через Makefile. `make check` требует UTF-8 режим Python на Windows (`PYTHONUTF8=1`),
|
||||
иначе `contracts/openapi.json` читается в cp1251. На машине автора `make` не установлен, поэтому все
|
||||
результаты выше получены прямыми командами из блока «Запуск из чистого clone»; цели Makefile —
|
||||
эквивалентные обёртки для POSIX-окружения.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Аппаратура: AMD Ryzen 7 7700 (8C/16T), 31.1 GiB RAM, Docker Desktop 29.4.3, Windows 11.
|
||||
Лимиты CPU/RAM контейнерам не задавались (учебный локальный стенд). Данные синтетические.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `docker stop` обслужившей API-реплики во время потока POST | LB уводит трафик на вторую реплику, подтверждённые сообщения не теряются, дублей нет | 2 реплики наблюдались до отказа; 112 успешных записей за 15 с, 0 ошибок; первая успешная запись после старта hook — 2.33 с; остановленная реплика отсутствует в последних ответах. В более ранних прогонах (до настройки `timeout connect`/health-check) было 58–59 записей и 2–4 ошибки переходного периода, дублей не возникало ни разу | восстановление 2.3 с (ранее 5.3–7.4 с); потерь подтверждённых данных нет |
|
||||
| Рестарт `postgres`, `seaweedfs`, `haproxy`, `api1`, `api2` с теми же volumes | сессия, чат, сообщения, метаданные и оригинал сохраняются | `PASS: persistence preserved confirmed data and existing session`; download после рестарта побайтно совпал | простой на время рестарта (~10–20 с); потерь нет |
|
||||
| Остановка S3 (`docker stop lab4-seaweedfs`) | attachment endpoints отвечают 503, метаданные остаются, данные не теряются | LB `/health/ready` → 503; upload и download → 503; прямой запрос к реплике: метаданные `200`, content `503` с `error_code=unavailable`; после старта S3 оригинал идентичен | после старта S3 готовность вернулась; потерь нет |
|
||||
| Повторная загрузка с тем же `Idempotency-Key` и другим body | 409 без второго сообщения | `409 conflict` в smoke | — |
|
||||
|
||||
Ограничения измерений: отдельный нагрузочный эксперимент (RPS/p50/p95/p99, пик RAM при ≥10
|
||||
одновременных загрузках) **не проводился**, так как он относится к уровню 4. Числа failover — это
|
||||
наблюдения нескольких локальных прогонов, а не гарантия SLA: LB, PostgreSQL и S3 развёрнуты на одном
|
||||
Docker-host, поэтому потеря хоста не выдерживается.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Заявлен и реализован уровень **3**. Инженерно закрыты отдельные практики уровня 4 (компенсация при
|
||||
сбое между S3 и PostgreSQL, отдельная ограниченная идентичность API, проверка приватности bucket,
|
||||
расширенные негативные тесты), но полный уровень 4 не заявляется: нет документированного reconciler
|
||||
с grace period и эксперимента с ≥10 одновременными загрузками и измерением пика RAM.
|
||||
- Изображение читается в память целиком, но не более 10 MiB на запрос; потоковая обработка тела
|
||||
загрузки и вынесение декодирования в отдельный процесс — следующий шаг.
|
||||
- Rate limiting (`429`) не реализован: схема допускает код, но рубрика уровня 3 его не требует.
|
||||
- Сессии хранятся в PostgreSQL; отдельный Redis/Valkey не используется.
|
||||
- Один Docker-host: отказ диска/host/сети не выдерживается; HA PostgreSQL и LB не заявляются.
|
||||
- Компенсация покрывает сбой между `PUT` в S3 и `COMMIT` в PostgreSQL. Полная сверка «БД ↔ S3»
|
||||
(reconciler с dry-run и grace period) — трек 2 уровня 5, он не реализован.
|
||||
- Ключи и пароли в `.env` — синтетические локальные значения; `.env` в Git не попадает.
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
"""Messenger API for laboratory 4."""
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Idempotent S3 bucket bootstrap using an administrative identity."""
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def required(name):
|
||||
value = os.environ.get(name)
|
||||
if not value:
|
||||
raise RuntimeError("required configuration is missing: " + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
endpoint = required("S3_ENDPOINT")
|
||||
region = os.environ.get("S3_REGION", "us-east-1")
|
||||
bucket = required("S3_BUCKET")
|
||||
access_key = required("S3_ADMIN_ACCESS_KEY_ID")
|
||||
secret_key = required("S3_ADMIN_SECRET_ACCESS_KEY")
|
||||
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=BotoConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=10,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
break
|
||||
except ClientError as error:
|
||||
code = str(error.response.get("Error", {}).get("Code", ""))
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
try:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
break
|
||||
except ClientError as create_error:
|
||||
create_code = str(
|
||||
create_error.response.get("Error", {}).get("Code", "")
|
||||
)
|
||||
if create_code in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
break
|
||||
last_error = create_error
|
||||
else:
|
||||
last_error = error
|
||||
except BotoCoreError as error:
|
||||
last_error = error
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError("S3 bucket bootstrap failed") from last_error
|
||||
time.sleep(1)
|
||||
|
||||
# SeaweedFS buckets are private by default and anonymous requests have no
|
||||
# grant. Some S3 implementations close the connection for unsupported
|
||||
# policy operations, so this best-effort cleanup must not fail bootstrap.
|
||||
try:
|
||||
client.delete_bucket_policy(Bucket=bucket)
|
||||
except (BotoCoreError, ClientError):
|
||||
pass
|
||||
sys.stderr.write("bucket is ready: %s\n" % bucket)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Runtime configuration loaded from environment variables or *_FILE secrets."""
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _secret(name, default=None, required=False):
|
||||
value = os.environ.get(name)
|
||||
if value is None:
|
||||
file_name = os.environ.get(name + "_FILE")
|
||||
if file_name:
|
||||
value = Path(file_name).read_text(encoding="utf-8").strip()
|
||||
if value is None:
|
||||
if required:
|
||||
raise RuntimeError("required configuration is missing: " + name)
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _int_secret(name, default):
|
||||
value = _secret(name)
|
||||
return default if value is None else int(value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
database_url: str
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
session_ttl_seconds: int
|
||||
idempotency_ttl_seconds: int
|
||||
instance_id: str
|
||||
cursor_secret: str
|
||||
host: str
|
||||
port: int
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
return cls(
|
||||
database_url=_secret("DATABASE_URL", required=True),
|
||||
s3_endpoint=_secret("S3_ENDPOINT", "http://s3:9000"),
|
||||
s3_region=_secret("S3_REGION", "us-east-1"),
|
||||
s3_bucket=_secret("S3_BUCKET", "messenger-attachments"),
|
||||
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=True),
|
||||
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=True),
|
||||
max_upload_bytes=_int_secret("MAX_UPLOAD_BYTES", 10_485_760),
|
||||
max_image_pixels=_int_secret("MAX_IMAGE_PIXELS", 40_000_000),
|
||||
session_ttl_seconds=_int_secret("SESSION_TTL_SECONDS", 3600),
|
||||
idempotency_ttl_seconds=_int_secret("IDEMPOTENCY_TTL_SECONDS", 86_400),
|
||||
instance_id=_secret("INSTANCE_ID", "api-local"),
|
||||
cursor_secret=_secret("CURSOR_SECRET", "local-dev-only-cursor-secret"),
|
||||
host=_secret("APP_HOST", "0.0.0.0"),
|
||||
port=_int_secret("APP_PORT", 8080),
|
||||
)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""PostgreSQL pool and idempotent schema bootstrap."""
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
MIGRATION_LOCK = 0x4C414234 # "LAB4"
|
||||
|
||||
SCHEMA = [
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL UNIQUE,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions(expires_at)",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
""",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members(user_id, chat_id)",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages(chat_id, created_at, id)",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
status TEXT NOT NULL,
|
||||
variants JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
error_code TEXT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments(chat_id)",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx ON message_attachments(attachment_id)",
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS idempotency (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
)
|
||||
""",
|
||||
"CREATE INDEX IF NOT EXISTS idempotency_created_idx ON idempotency(created_at)",
|
||||
]
|
||||
|
||||
|
||||
def wait_for_database(dsn, timeout_seconds=90):
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
last_error = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with psycopg.connect(dsn, connect_timeout=3) as connection:
|
||||
connection.execute("SELECT 1")
|
||||
return
|
||||
except psycopg.Error as error:
|
||||
last_error = error
|
||||
time.sleep(1)
|
||||
raise RuntimeError("PostgreSQL did not become available") from last_error
|
||||
|
||||
|
||||
def create_pool(dsn):
|
||||
return ConnectionPool(
|
||||
conninfo=dsn,
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=10,
|
||||
kwargs={"row_factory": dict_row},
|
||||
open=False,
|
||||
)
|
||||
|
||||
|
||||
def migrate(pool):
|
||||
with pool.connection() as connection:
|
||||
with connection.transaction():
|
||||
with connection.cursor() as cursor:
|
||||
cursor.execute("SELECT pg_advisory_xact_lock(%s)", (MIGRATION_LOCK,))
|
||||
for statement in SCHEMA:
|
||||
cursor.execute(statement)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Bounded image inspection: sniff MIME, enforce pixel limit, verify decoding."""
|
||||
from io import BytesIO
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
# We enforce the configured limit explicitly before any full decode. Disabling
|
||||
# Pillow's own heuristic keeps the error deterministic for the API contract.
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ImageTooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def sniff_content_type(data):
|
||||
if data.startswith(PNG_MAGIC):
|
||||
return "image/png"
|
||||
if data.startswith(JPEG_MAGIC):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def inspect_image(data, declared_content_type, max_pixels):
|
||||
"""Validate bytes and return (actual_content_type, width, height)."""
|
||||
actual = sniff_content_type(data)
|
||||
if actual is None or actual != declared_content_type:
|
||||
raise InvalidImage("unsupported or mismatched image content")
|
||||
|
||||
try:
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
width, height = image.size
|
||||
if width <= 0 or height <= 0:
|
||||
raise InvalidImage("invalid image dimensions")
|
||||
if width * height > max_pixels:
|
||||
raise ImageTooLarge("decoded pixel limit exceeded")
|
||||
if image.format not in ("PNG", "JPEG"):
|
||||
raise InvalidImage("unsupported image format")
|
||||
image.verify()
|
||||
|
||||
# verify() checks structure; load() catches truncated payloads and
|
||||
# corrupt compressed streams that would otherwise reach the client.
|
||||
with Image.open(BytesIO(data)) as image:
|
||||
image.load()
|
||||
except ImageTooLarge:
|
||||
raise
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
||||
raise InvalidImage("corrupt image") from None
|
||||
|
||||
return actual, width, height
|
||||
@@ -0,0 +1,4 @@
|
||||
from .server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Password hashing, opaque sessions and signed pagination cursors."""
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
|
||||
SCRYPT_N = 2 ** 14
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
SCRYPT_DKLEN = 32
|
||||
SCRYPT_MAXMEM = 64 * 1024 * 1024
|
||||
|
||||
DUMMY_PASSWORD_HASH = None
|
||||
|
||||
|
||||
def _b64encode(value):
|
||||
return base64.urlsafe_b64encode(value).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _b64decode(value):
|
||||
padding = "=" * (-len(value) % 4)
|
||||
return base64.urlsafe_b64decode(value + padding)
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
salt = secrets.token_bytes(16)
|
||||
digest = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=SCRYPT_N,
|
||||
r=SCRYPT_R,
|
||||
p=SCRYPT_P,
|
||||
dklen=SCRYPT_DKLEN,
|
||||
maxmem=SCRYPT_MAXMEM,
|
||||
)
|
||||
return "scrypt${}${}${}${}${}".format(
|
||||
SCRYPT_N, SCRYPT_R, SCRYPT_P, _b64encode(salt), _b64encode(digest)
|
||||
)
|
||||
|
||||
|
||||
def verify_password(password, stored):
|
||||
try:
|
||||
algorithm, n, r, p, salt, digest = stored.split("$")
|
||||
if algorithm != "scrypt":
|
||||
return False
|
||||
expected = _b64decode(digest)
|
||||
actual = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=_b64decode(salt),
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(expected),
|
||||
maxmem=SCRYPT_MAXMEM,
|
||||
)
|
||||
return hmac.compare_digest(actual, expected)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def dummy_password_hash():
|
||||
global DUMMY_PASSWORD_HASH
|
||||
if DUMMY_PASSWORD_HASH is None:
|
||||
DUMMY_PASSWORD_HASH = hash_password(secrets.token_urlsafe(24))
|
||||
return DUMMY_PASSWORD_HASH
|
||||
|
||||
|
||||
def new_session_token():
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def _signature(secret, payload):
|
||||
return hmac.new(secret.encode("utf-8"), payload, hashlib.sha256).hexdigest()[:32]
|
||||
|
||||
|
||||
def encode_cursor(secret, payload):
|
||||
raw = json.dumps(payload, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
return _b64encode(raw) + "." + _signature(secret, raw)
|
||||
|
||||
|
||||
def decode_cursor(secret, value):
|
||||
if not value or "." not in value:
|
||||
raise ValueError("invalid cursor")
|
||||
encoded, signature = value.rsplit(".", 1)
|
||||
raw = _b64decode(encoded)
|
||||
if not hmac.compare_digest(_signature(secret, raw), signature):
|
||||
raise ValueError("invalid cursor")
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("invalid cursor")
|
||||
return payload
|
||||
+1074
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
"""Private S3-compatible object storage client."""
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, endpoint, region, bucket, access_key_id, secret_access_key):
|
||||
self.bucket = bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=region,
|
||||
aws_access_key_id=access_key_id,
|
||||
aws_secret_access_key=secret_access_key,
|
||||
config=BotoConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
# Fail fast: a stalled storage call must not hold an API thread
|
||||
# for the whole client timeout, and retries are the client's job.
|
||||
retries={"max_attempts": 1, "mode": "standard"},
|
||||
),
|
||||
)
|
||||
|
||||
def ready(self):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return True
|
||||
except (BotoCoreError, ClientError):
|
||||
return False
|
||||
|
||||
def put(self, key, data, content_type):
|
||||
self.client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
ContentLength=len(data),
|
||||
)
|
||||
|
||||
def get(self, key):
|
||||
return self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
def delete(self, key):
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
|
||||
def is_missing_object(error):
|
||||
if not isinstance(error, ClientError):
|
||||
return False
|
||||
code = str(error.response.get("Error", {}).get("Code", ""))
|
||||
status = error.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
return code in ("NoSuchKey", "NotFound", "404") or status == 404
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
x-api-environment: &api-environment
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}@postgres:5432/${POSTGRES_DB:-messenger}
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
CURSOR_SECRET: ${CURSOR_SECRET:?CURSOR_SECRET is required}
|
||||
PYTHONUNBUFFERED: "1"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
container_name: lab4-postgres
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is required}
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
networks: [lab4]
|
||||
|
||||
seaweedfs:
|
||||
image: chrislusf/seaweedfs:3.71
|
||||
container_name: lab4-seaweedfs
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- |
|
||||
mkdir -p /tmp/seaweedfs
|
||||
cat > /tmp/seaweedfs/s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [{"accessKey": "${S3_ADMIN_ACCESS_KEY_ID}", "secretKey": "${S3_ADMIN_SECRET_ACCESS_KEY}"}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [{"accessKey": "${S3_ACCESS_KEY_ID}", "secretKey": "${S3_SECRET_ACCESS_KEY}"}],
|
||||
"actions": ["Read", "Write", "List", "Tagging"]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
exec weed server -dir=/data -filer -s3 -s3.port=8333 -s3.config=/tmp/seaweedfs/s3.json
|
||||
environment:
|
||||
S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?S3_ADMIN_ACCESS_KEY_ID is required}
|
||||
S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?S3_ADMIN_SECRET_ACCESS_KEY is required}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?S3_ACCESS_KEY_ID is required}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?S3_SECRET_ACCESS_KEY is required}
|
||||
volumes:
|
||||
- seaweeddata:/data
|
||||
restart: unless-stopped
|
||||
networks: [lab4]
|
||||
|
||||
bootstrap:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: lab4-messenger-api:local
|
||||
container_name: lab4-bootstrap
|
||||
depends_on:
|
||||
seaweedfs:
|
||||
condition: service_started
|
||||
environment:
|
||||
S3_ENDPOINT: http://seaweedfs:8333
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?S3_ADMIN_ACCESS_KEY_ID is required}
|
||||
S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?S3_ADMIN_SECRET_ACCESS_KEY is required}
|
||||
command: ["python", "-m", "app.bootstrap_s3"]
|
||||
restart: "no"
|
||||
networks: [lab4]
|
||||
|
||||
api1:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: lab4-messenger-api:local
|
||||
container_name: lab4-api1
|
||||
environment:
|
||||
<<: *api-environment
|
||||
INSTANCE_ID: api1
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
networks: [lab4]
|
||||
|
||||
api2:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: lab4-messenger-api:local
|
||||
container_name: lab4-api2
|
||||
environment:
|
||||
<<: *api-environment
|
||||
INSTANCE_ID: api2
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
bootstrap:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2)"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
networks: [lab4]
|
||||
|
||||
haproxy:
|
||||
image: haproxy:3.0.8-alpine
|
||||
container_name: lab4-haproxy
|
||||
depends_on:
|
||||
api1:
|
||||
condition: service_healthy
|
||||
api2:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
# Loopback only: the stand is local, and binding IPv4 explicitly avoids a
|
||||
# broken IPv6 path when a client resolves "localhost" to ::1 first.
|
||||
- "127.0.0.1:${LAB4_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
restart: unless-stopped
|
||||
networks: [lab4]
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
seaweeddata:
|
||||
|
||||
networks:
|
||||
lab4:
|
||||
driver: bridge
|
||||
@@ -0,0 +1 @@
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,31 @@
|
||||
Table "public.attachments"
|
||||
Column | Type | Collation | Nullable | Default
|
||||
--------------+--------------------------+-----------+----------+-------------
|
||||
id | uuid | | not null |
|
||||
chat_id | uuid | | not null |
|
||||
owner_id | uuid | | not null |
|
||||
filename | text | | not null |
|
||||
content_type | text | | not null |
|
||||
size_bytes | bigint | | not null |
|
||||
sha256 | text | | not null |
|
||||
object_key | text | | not null |
|
||||
status | text | | not null |
|
||||
variants | jsonb | | not null | '[]'::jsonb
|
||||
error_code | text | | |
|
||||
created_at | timestamp with time zone | | not null |
|
||||
updated_at | timestamp with time zone | | not null |
|
||||
Indexes:
|
||||
"attachments_pkey" PRIMARY KEY, btree (id)
|
||||
"attachments_chat_idx" btree (chat_id)
|
||||
"attachments_object_key_key" UNIQUE CONSTRAINT, btree (object_key)
|
||||
Foreign-key constraints:
|
||||
"attachments_chat_id_fkey" FOREIGN KEY (chat_id) REFERENCES chats(id) ON DELETE CASCADE
|
||||
"attachments_owner_id_fkey" FOREIGN KEY (owner_id) REFERENCES users(id)
|
||||
Referenced by:
|
||||
TABLE "message_attachments" CONSTRAINT "message_attachments_attachment_id_fkey" FOREIGN KEY (attachment_id) REFERENCES attachments(id)
|
||||
|
||||
attachments | object_keys
|
||||
-------------+-------------
|
||||
7 | 7
|
||||
(1 row)
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
SERVICE NAME STATUS PORTS
|
||||
api1 lab4-api1 Up About a minute (healthy) 8080/tcp
|
||||
api2 lab4-api2 Up About a minute (healthy) 8080/tcp
|
||||
haproxy lab4-haproxy Up About a minute 127.0.0.1:8081->8080/tcp
|
||||
postgres lab4-postgres Up About a minute (healthy) 5432/tcp
|
||||
seaweedfs lab4-seaweedfs Up 45 seconds 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.328}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,14 @@
|
||||
PASS: oversized upload returns 413 before decoding
|
||||
PASS: corrupt image returns 415
|
||||
PASS: MIME mismatch returns 415
|
||||
PASS: unsupported declared type returns 415
|
||||
PASS: path-like and control-character filenames return 400
|
||||
PASS: pixel limit returns 413
|
||||
PASS: attachment from another chat cannot be attached
|
||||
PASS: unknown attachment id returns 404
|
||||
PASS: outsider cannot read metadata or bytes
|
||||
PASS: chat member downloads identical bytes
|
||||
PASS: real JPEG upload roundtrip
|
||||
PASS: attachment_ids survive message history
|
||||
|
||||
PASS: all negative attachment scenarios
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"objects_in_bucket": 7,
|
||||
"first_key": "chats/188fd76c-74c0-4861-a521-8b020b1430e8/95a16c4a-e8c0-4da6-9b7e-581f96762c27/original",
|
||||
"anonymous_object_get_status": 403,
|
||||
"anonymous_bucket_list_status": 403
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
stage=baseline attachment=9aabb7d3-764a-4d15-b769-21ea2810beb3 sha256=99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
stage=s3_down lb_readiness_status=503
|
||||
stage=s3_down lb_upload_status=503
|
||||
stage=s3_down lb_download_status=503
|
||||
stage=s3_down api_replica_probe={"metadata": {"status": 200}, "content": {"status": 503, "error_code": "unavailable", "has_request_id": true}}
|
||||
stage=s3_restored readiness=200 download=identical
|
||||
PASS: S3 outage surfaced as 503 and the original survived the restart
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,17 @@
|
||||
.venv\Scripts\python : test_signed_cursor_roundtrip_and_tampering (test_core.CursorTests) ... ok
|
||||
строка:1 знак:382
|
||||
+ ... ntract.txt; .venv\Scripts\python -m unittest discover -s student-test ...
|
||||
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
+ CategoryInfo : NotSpecified: (test_signed_cur...orTests) ... ok:String) [], RemoteException
|
||||
+ FullyQualifiedErrorId : NativeCommandError
|
||||
|
||||
test_corrupt_and_oversized (test_core.ImageTests) ... ok
|
||||
test_filename_rules (test_core.ImageTests) ... ok
|
||||
test_fixture_png (test_core.ImageTests) ... ok
|
||||
test_jpeg_and_mime_mismatch (test_core.ImageTests) ... ok
|
||||
test_password_roundtrip (test_core.PasswordTests) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 6 tests in 0.099s
|
||||
|
||||
OK
|
||||
@@ -0,0 +1 @@
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -5,3 +5,26 @@
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
|
||||
## Реализованная топология (lab4, Compose)
|
||||
|
||||
| Роль | Service | Образ (закреплён) | Readiness | Volume | Порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| api | `api1`, `api2` | сборка из `Dockerfile`, `lab4-messenger-api:local` | `GET /health/live`, плюс `/health/ready` (PostgreSQL + S3) для LB | нет локального состояния | `8080` только внутри сети | `DATABASE_URL`, `S3_ACCESS_KEY_ID`/`S3_SECRET_ACCESS_KEY` из `.env` |
|
||||
| postgres | `postgres` | `postgres:16.4-alpine` | `pg_isready` | `pgdata:/var/lib/postgresql/data` | `5432` внутри сети | `POSTGRES_*` из `.env` |
|
||||
| s3 | `seaweedfs` | `chrislusf/seaweedfs:3.71` | TCP 8333 (проверяется через `bootstrap` и readiness API) | `seaweeddata:/data` | `8333` внутри сети | `s3.json` создаётся из `S3_*` переменных при старте |
|
||||
| load_balancer | `haproxy` | `haproxy:3.0.8-alpine` | `GET /health/ready` на каждом backend | — | `127.0.0.1:${LAB4_PORT}:8080` | — |
|
||||
| bootstrap (one-shot) | `bootstrap` | `lab4-messenger-api:local` | завершение с кодом 0 | — | — | `S3_ADMIN_*` только для создания bucket |
|
||||
|
||||
Конфигурации: `infra/haproxy/haproxy.cfg` — балансировка и health-check. `infra/minio/` не используется
|
||||
(выбран SeaweedFS).
|
||||
|
||||
Bootstrap:
|
||||
|
||||
1. PostgreSQL: DDL в `app/db.py` выполняется каждой репликой при старте под
|
||||
`pg_advisory_xact_lock`, все операторы `IF NOT EXISTS` — повторный запуск безопасен.
|
||||
2. S3: `app/bootstrap_s3.py` — `head_bucket`, при `404` `create_bucket`, повторный запуск возвращает 0.
|
||||
Bucket приватный: анонимные запросы получают `403` (см. `scripts/verify_private_bucket.py`).
|
||||
|
||||
Секреты передаются только через переменные окружения из локального `.env` (в Git не попадает);
|
||||
поддерживаются `*_FILE`-варианты для запуска вне Compose.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 4096
|
||||
|
||||
# Docker's embedded DNS. Re-resolving the replica names periodically prevents a
|
||||
# stale address from being used if a container is recreated with another IP
|
||||
# (a name resolved once at startup can otherwise point to the wrong replica).
|
||||
resolvers docker
|
||||
nameserver dns1 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 5s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold refused 1s
|
||||
hold other 1s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
option redispatch
|
||||
# Close the server-side connection after each response so that roundrobin
|
||||
# distributes every request instead of reusing one idle backend connection.
|
||||
option http-server-close
|
||||
retries 2
|
||||
# A stopped container's IP silently drops packets, so a short connect
|
||||
# timeout bounds how long a request waits before redispatch to a live peer.
|
||||
timeout connect 500ms
|
||||
timeout client 60s
|
||||
timeout server 60s
|
||||
|
||||
frontend public_http
|
||||
bind *:8080
|
||||
default_backend api_backend
|
||||
|
||||
backend api_backend
|
||||
balance roundrobin
|
||||
option httpchk GET /health/ready
|
||||
http-check expect status 200
|
||||
# Fast detection: a stopped replica must leave rotation within a couple of
|
||||
# seconds, otherwise in-flight writes fail while the LB still routes to it.
|
||||
default-server check inter 1s fastinter 200ms downinter 500ms fall 2 rise 1
|
||||
server api1 api1:8080 check resolvers docker init-addr none
|
||||
server api2 api2:8080 check resolvers docker init-addr none
|
||||
@@ -0,0 +1,4 @@
|
||||
boto3==1.35.36
|
||||
Pillow==11.0.0
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg_pool==3.2.3
|
||||
@@ -0,0 +1,30 @@
|
||||
"""In-container probe: status and error code of attachment endpoints.
|
||||
|
||||
Used as evidence that a replica answers with the common JSON error format while
|
||||
S3 is unavailable (the load balancer may answer 503 on its own before this).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
TOKEN = os.environ["TOKEN"]
|
||||
ATTACHMENT_ID = os.environ["ATTACHMENT_ID"]
|
||||
|
||||
|
||||
def probe(url):
|
||||
request = urllib.request.Request(url, headers={"Authorization": "Bearer " + TOKEN})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return {"status": response.status}
|
||||
except urllib.error.HTTPError as error:
|
||||
payload = json.loads(error.read().decode("utf-8"))
|
||||
return {
|
||||
"status": error.code,
|
||||
"error_code": payload["error"]["code"],
|
||||
"has_request_id": bool(error.headers.get("X-Request-Id")),
|
||||
}
|
||||
|
||||
|
||||
base = "http://127.0.0.1:8080/api/v1/attachments/" + ATTACHMENT_ID
|
||||
print(json.dumps({"metadata": probe(base), "content": probe(base + "/content")}))
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Explicit evidence scenario: behaviour when the S3 backend is unavailable.
|
||||
|
||||
Stops the S3 service, observes 503 responses through the load balancer and from
|
||||
an API replica, then starts it again and verifies that the previously uploaded
|
||||
original is still downloadable with the same checksum.
|
||||
This script is NOT part of `make test`: it deliberately stops infrastructure.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def compose(*arguments):
|
||||
command = ["docker", "compose", "--env-file", ".env", *arguments]
|
||||
result = subprocess.run(command, cwd=str(ROOT), capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("compose command failed: " + " ".join(arguments))
|
||||
return result
|
||||
|
||||
|
||||
def raw_status(method, url, token=None, body=None, content_type=None):
|
||||
headers = {}
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
if content_type:
|
||||
headers["Content-Type"] = content_type
|
||||
request = urllib.request.Request(url, data=body, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
|
||||
|
||||
def main():
|
||||
base = os.environ.get("BASE_URL", "http://127.0.0.1:8081")
|
||||
client = Client(base)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
print(
|
||||
"stage=baseline attachment=%s sha256=%s"
|
||||
% (attachment["id"], hashlib.sha256(content).hexdigest())
|
||||
)
|
||||
|
||||
compose("cp", "scripts/check_api_error.py", "api1:/tmp/check_api_error.py")
|
||||
compose("stop", "seaweedfs")
|
||||
try:
|
||||
print("stage=s3_down lb_readiness_status=%d" % raw_status("GET", base + "/health/ready"))
|
||||
print(
|
||||
"stage=s3_down lb_upload_status=%d"
|
||||
% raw_status(
|
||||
"POST",
|
||||
base + "/api/v1/chats/" + chat["id"] + "/attachments?filename=after-outage.png",
|
||||
token=alice["token"],
|
||||
body=content,
|
||||
content_type="image/png",
|
||||
)
|
||||
)
|
||||
print(
|
||||
"stage=s3_down lb_download_status=%d"
|
||||
% raw_status(
|
||||
"GET",
|
||||
base + "/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
token=bob["token"],
|
||||
)
|
||||
)
|
||||
probe = compose(
|
||||
"exec",
|
||||
"-T",
|
||||
"-e",
|
||||
"TOKEN=" + bob["token"],
|
||||
"-e",
|
||||
"ATTACHMENT_ID=" + attachment["id"],
|
||||
"api1",
|
||||
"python",
|
||||
"/tmp/check_api_error.py",
|
||||
)
|
||||
print("stage=s3_down api_replica_probe=" + probe.stdout.strip())
|
||||
finally:
|
||||
compose("start", "seaweedfs")
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
if client.call("GET", "/health/ready", schema="Health")["status"] == 200:
|
||||
break
|
||||
except Failure:
|
||||
pass
|
||||
if time.monotonic() >= deadline:
|
||||
raise SystemExit("stack did not become ready within 90 seconds")
|
||||
time.sleep(1)
|
||||
|
||||
restored = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
||||
if restored["bytes"] != content:
|
||||
raise SystemExit("original changed after S3 restart")
|
||||
print("stage=s3_restored readiness=200 download=identical")
|
||||
print("PASS: S3 outage surfaced as 503 and the original survived the restart")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,29 @@
|
||||
@echo off
|
||||
rem Windows equivalent of restart.sh for local evidence collection.
|
||||
rem Restarts every stateful service and the API replicas without removing volumes.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
if not exist ".env" (
|
||||
echo missing .env; copy .env.example and fill local values 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
|
||||
set "LAB4_PORT="
|
||||
for /f "usebackq tokens=1,* delims==" %%a in (".env") do (
|
||||
if /i "%%a"=="LAB4_PORT" set "LAB4_PORT=%%b"
|
||||
)
|
||||
if "%LAB4_PORT%"=="" set "LAB4_PORT=8080"
|
||||
|
||||
docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2
|
||||
if errorlevel 1 exit /b 1
|
||||
|
||||
rem /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous
|
||||
rem path is usable again. Wait for it before reporting success.
|
||||
rem ping is used instead of timeout: timeout fails when stdin is redirected.
|
||||
for /l %%i in (1,1,90) do (
|
||||
powershell -NoProfile -Command "try { if ((Invoke-WebRequest -Uri 'http://127.0.0.1:%LAB4_PORT%/health/ready' -UseBasicParsing -TimeoutSec 3).StatusCode -eq 200) { exit 0 } else { exit 1 } } catch { exit 1 }" >nul 2>&1
|
||||
if not errorlevel 1 exit /b 0
|
||||
ping -n 2 127.0.0.1 >nul
|
||||
)
|
||||
echo stack did not become ready within 90 seconds 1>&2
|
||||
exit /b 1
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Restart every process that owns state, without removing volumes or re-seeding.
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
echo "missing .env; copy .env.example and fill local values" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
docker compose --env-file .env restart postgres seaweedfs haproxy api1 api2
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
. ./.env
|
||||
port="${LAB4_PORT:-8080}"
|
||||
|
||||
# /health/ready covers PostgreSQL and S3, so a 200 means the whole synchronous
|
||||
# path is usable again. Wait for it before reporting success to the caller.
|
||||
i=0
|
||||
while [ "$i" -lt 90 ]; do
|
||||
if curl -fsS "http://127.0.0.1:${port}/health/ready" >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
i=$((i + 1))
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "stack did not become ready within 90 seconds" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,14 @@
|
||||
@echo off
|
||||
rem Windows equivalent of stop-one.sh: stop exactly one API replica and leave it stopped.
|
||||
setlocal
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo test provides target API instance alias 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
if /i not "%TARGET_INSTANCE_ID%"=="api1" if /i not "%TARGET_INSTANCE_ID%"=="api2" (
|
||||
echo unexpected API instance alias 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
docker stop lab4-%TARGET_INSTANCE_ID% >nul
|
||||
if errorlevel 1 exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api1|api2)
|
||||
;;
|
||||
*)
|
||||
echo "unexpected API instance alias" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# Leave exactly one API replica stopped. The test restores it explicitly later.
|
||||
docker stop "lab4-${TARGET_INSTANCE_ID}" >/dev/null
|
||||
@@ -0,0 +1,50 @@
|
||||
"""Evidence helper: the bucket must not serve objects to anonymous clients.
|
||||
|
||||
Runs inside an API container (it needs the API's S3 credentials and network
|
||||
access to the S3 endpoint). Prints a small JSON summary without secrets.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from urllib.parse import quote
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
|
||||
|
||||
def main():
|
||||
endpoint = os.environ["S3_ENDPOINT"].rstrip("/")
|
||||
bucket = os.environ["S3_BUCKET"]
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=endpoint,
|
||||
region_name=os.environ.get("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],
|
||||
aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],
|
||||
config=BotoConfig(signature_version="s3v4", s3={"addressing_style": "path"}),
|
||||
)
|
||||
objects = client.list_objects_v2(Bucket=bucket).get("Contents", [])
|
||||
summary = {"objects_in_bucket": len(objects), "first_key": None}
|
||||
if not objects:
|
||||
print(json.dumps(summary))
|
||||
return 0
|
||||
key = objects[0]["Key"]
|
||||
summary["first_key"] = key
|
||||
|
||||
def anonymous(url):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code
|
||||
|
||||
summary["anonymous_object_get_status"] = anonymous(endpoint + "/" + bucket + "/" + quote(key, safe="/"))
|
||||
summary["anonymous_bucket_list_status"] = anonymous(endpoint + "/" + bucket + "?list-type=2")
|
||||
print(json.dumps(summary, indent=2))
|
||||
return 0 if summary["anonymous_object_get_status"] >= 400 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,196 @@
|
||||
"""Negative attachment scenarios against a running stand (lab4).
|
||||
|
||||
Run: python student-tests/test_api_negative.py --base-url http://localhost:8081
|
||||
Creates only synthetic users, chats and files.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
MAX_UPLOAD_BYTES = 10_485_760
|
||||
MAX_PIXELS = 40_000_000
|
||||
|
||||
|
||||
def upload(client, actor, chat, content, content_type, filename="photo.png", expected=201):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(filename, safe="")
|
||||
return client.call(
|
||||
"POST",
|
||||
path,
|
||||
expected,
|
||||
actor=actor,
|
||||
raw=content,
|
||||
headers={"Content-Type": content_type},
|
||||
schema="Attachment" if expected == 201 else None,
|
||||
)
|
||||
|
||||
|
||||
def base_url():
|
||||
for index, argument in enumerate(sys.argv):
|
||||
if argument == "--base-url" and index + 1 < len(sys.argv):
|
||||
return sys.argv[index + 1]
|
||||
if argument.startswith("--base-url="):
|
||||
return argument.split("=", 1)[1]
|
||||
return os.environ.get("BASE_URL", "http://127.0.0.1:8081")
|
||||
|
||||
|
||||
def main():
|
||||
base = base_url()
|
||||
client = Client(base)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
other = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Второй чат", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
sample = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
failures = []
|
||||
|
||||
def scenario(name, function):
|
||||
try:
|
||||
function()
|
||||
print("PASS:", name)
|
||||
except AssertionError as error:
|
||||
failures.append(name + ": " + str(error))
|
||||
print("FAIL:", name, "-", error)
|
||||
|
||||
def too_large():
|
||||
payload = b"\x89PNG\r\n\x1a\n" + b"\x00" * (MAX_UPLOAD_BYTES + 1 - 8)
|
||||
response = upload(client, alice, chat, payload, "image/png", expected=413)
|
||||
check(response["json"]["error"]["code"] == "payload_too_large", "wrong error code")
|
||||
|
||||
def corrupt():
|
||||
upload(client, alice, chat, b"\x89PNG\r\n\x1a\n" + b"broken" * 500, "image/png", expected=415)
|
||||
|
||||
def mime_mismatch():
|
||||
upload(client, alice, chat, sample, "image/jpeg", expected=415)
|
||||
|
||||
def unsupported_type():
|
||||
upload(client, alice, chat, sample, "image/gif", expected=415)
|
||||
|
||||
def dangerous_names():
|
||||
for bad in ("a/b.png", "a\b.png", "a\x00b.png", "a\nb.png"):
|
||||
response = upload(client, alice, chat, sample, "image/png", filename=bad, expected=400)
|
||||
check(response["json"]["error"]["code"] == "invalid_request", "wrong error code")
|
||||
|
||||
def pixel_limit():
|
||||
buffer = BytesIO()
|
||||
side_width, side_height = 7000, 6000 # 42 Mpx > configured 40 Mpx limit
|
||||
Image.new("L", (side_width, side_height), 128).save(buffer, "PNG", optimize=True)
|
||||
payload = buffer.getvalue()
|
||||
check(len(payload) <= MAX_UPLOAD_BYTES, "fixture exceeds byte limit, adjust the test")
|
||||
check(side_width * side_height > MAX_PIXELS, "fixture must exceed the pixel limit")
|
||||
upload(client, alice, chat, payload, "image/png", filename="huge.png", expected=413)
|
||||
|
||||
def cross_chat_attachment():
|
||||
attachment = upload(client, alice, chat, sample, "image/png")["json"]
|
||||
response = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + other["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
check(response["json"]["error"]["code"] == "not_found", "wrong error code")
|
||||
page = client.call(
|
||||
"GET", "/api/v1/chats/" + other["id"] + "/messages?limit=100", actor=alice, schema="MessagePage"
|
||||
)["json"]
|
||||
check(
|
||||
all(attachment["id"] not in item["attachment_ids"] for item in page["items"]),
|
||||
"rejected attachment leaked into a message",
|
||||
)
|
||||
|
||||
def unknown_attachment():
|
||||
client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
404,
|
||||
actor=alice,
|
||||
body={"text": "нет такого файла", "attachment_ids": [str(uuid.uuid4())]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
def outsider_access():
|
||||
attachment = upload(client, alice, chat, sample, "image/png")["json"]
|
||||
for suffix in ("", "/content"):
|
||||
client.call("GET", "/api/v1/attachments/" + attachment["id"] + suffix, 404, actor=eve)
|
||||
|
||||
def member_access_and_checksum():
|
||||
import hashlib
|
||||
|
||||
attachment = upload(client, alice, chat, sample, "image/png")["json"]
|
||||
check(attachment["sha256"] == hashlib.sha256(sample).hexdigest(), "checksum mismatch")
|
||||
content = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
||||
check(content["bytes"] == sample, "download differs from upload")
|
||||
check(content["headers"].get_content_type() == "image/png", "content type mismatch")
|
||||
|
||||
def real_jpeg_roundtrip():
|
||||
import hashlib
|
||||
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (640, 480), "navy").save(buffer, "JPEG", quality=90)
|
||||
payload = buffer.getvalue()
|
||||
attachment = upload(client, alice, chat, payload, "image/jpeg", filename="photo.jpg")["json"]
|
||||
check(attachment["status"] == "ready", "JPEG was not accepted")
|
||||
check(attachment["content_type"] == "image/jpeg", "JPEG content type mismatch")
|
||||
check(attachment["size_bytes"] == len(payload), "JPEG size mismatch")
|
||||
check(attachment["sha256"] == hashlib.sha256(payload).hexdigest(), "JPEG checksum mismatch")
|
||||
content = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob)
|
||||
check(content["bytes"] == payload, "JPEG download differs from upload")
|
||||
check(content["headers"].get_content_type() == "image/jpeg", "JPEG download type mismatch")
|
||||
|
||||
def attachment_ids_survive_history():
|
||||
attachment = upload(client, alice, chat, sample, "image/png")["json"]
|
||||
created = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats/" + chat["id"] + "/messages",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"text": "", "attachment_ids": [attachment["id"]]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex},
|
||||
schema="Message",
|
||||
)["json"]
|
||||
page = client.call(
|
||||
"GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage"
|
||||
)["json"]
|
||||
found = [item for item in page["items"] if item["id"] == created["id"]]
|
||||
check(len(found) == 1, "message with attachment is missing from history")
|
||||
check(found[0]["attachment_ids"] == [attachment["id"]], "history lost attachment_ids")
|
||||
check(found[0] == created, "history item differs from the POST response")
|
||||
|
||||
scenario("oversized upload returns 413 before decoding", too_large)
|
||||
scenario("corrupt image returns 415", corrupt)
|
||||
scenario("MIME mismatch returns 415", mime_mismatch)
|
||||
scenario("unsupported declared type returns 415", unsupported_type)
|
||||
scenario("path-like and control-character filenames return 400", dangerous_names)
|
||||
scenario("pixel limit returns 413", pixel_limit)
|
||||
scenario("attachment from another chat cannot be attached", cross_chat_attachment)
|
||||
scenario("unknown attachment id returns 404", unknown_attachment)
|
||||
scenario("outsider cannot read metadata or bytes", outsider_access)
|
||||
scenario("chat member downloads identical bytes", member_access_and_checksum)
|
||||
scenario("real JPEG upload roundtrip", real_jpeg_roundtrip)
|
||||
scenario("attachment_ids survive message history", attachment_ids_survive_history)
|
||||
|
||||
if failures:
|
||||
print("\nFAILED:", len(failures))
|
||||
for failure in failures:
|
||||
print(" -", failure)
|
||||
raise SystemExit(1)
|
||||
print("\nPASS: all attachment scenarios (limits, MIME, access, roundtrip, history)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Small dependency-light checks for pure validation helpers.
|
||||
|
||||
Run with: python -m unittest discover -s student-tests -v
|
||||
"""
|
||||
import sys
|
||||
import unittest
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.images import ImageTooLarge, InvalidImage, inspect_image # noqa: E402
|
||||
from app.security import ( # noqa: E402
|
||||
decode_cursor,
|
||||
encode_cursor,
|
||||
hash_password,
|
||||
verify_password,
|
||||
)
|
||||
from app.server import Handler # noqa: E402
|
||||
|
||||
|
||||
class PasswordTests(unittest.TestCase):
|
||||
def test_password_roundtrip(self):
|
||||
stored = hash_password("T3st-abcdefghijkl")
|
||||
self.assertTrue(verify_password("T3st-abcdefghijkl", stored))
|
||||
self.assertFalse(verify_password("wrong-password", stored))
|
||||
self.assertNotIn("T3st-abcdefghijkl", stored)
|
||||
|
||||
|
||||
class CursorTests(unittest.TestCase):
|
||||
def test_signed_cursor_roundtrip_and_tampering(self):
|
||||
value = encode_cursor("secret", {"kind": "messages", "id": "1"})
|
||||
self.assertEqual(decode_cursor("secret", value)["id"], "1")
|
||||
with self.assertRaises(ValueError):
|
||||
decode_cursor("other-secret", value)
|
||||
with self.assertRaises(ValueError):
|
||||
decode_cursor("secret", value + "x")
|
||||
|
||||
|
||||
class ImageTests(unittest.TestCase):
|
||||
def test_fixture_png(self):
|
||||
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
self.assertEqual(inspect_image(data, "image/png", 40_000_000), ("image/png", 320, 200))
|
||||
|
||||
def test_jpeg_and_mime_mismatch(self):
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (10, 10), "red").save(buffer, "JPEG")
|
||||
data = buffer.getvalue()
|
||||
self.assertEqual(inspect_image(data, "image/jpeg", 40_000_000)[0], "image/jpeg")
|
||||
with self.assertRaises(InvalidImage):
|
||||
inspect_image(data, "image/png", 40_000_000)
|
||||
|
||||
def test_corrupt_and_oversized(self):
|
||||
with self.assertRaises(InvalidImage):
|
||||
inspect_image(b"not an image", "image/png", 40_000_000)
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (10, 10), "red").save(buffer, "PNG")
|
||||
with self.assertRaises(ImageTooLarge):
|
||||
inspect_image(buffer.getvalue(), "image/png", 99)
|
||||
|
||||
def test_filename_rules(self):
|
||||
self.assertTrue(Handler._valid_filename("photo.png"))
|
||||
for bad in ("", "a/b.png", "a\\b.png", "a\x00b", "a\nb"):
|
||||
self.assertFalse(Handler._valid_filename(bad), repr(bad))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user