diff --git a/.env.example b/.env.example index d99d26a2..e342cdf0 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,10 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Пример только для локального учебного стенда. Не публикуйте файл .env. +POSTGRES_PASSWORD=local_pg_password_2026 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ROOT_USER=local_lab4_root +S3_ROOT_PASSWORD=change_admin_password_2026 +S3_ACCESS_KEY_ID=local_lab4_api +S3_SECRET_ACCESS_KEY=change_scoped_password_2026 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..e6e63eb2 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,4 +1,4 @@ -## Лабораторная 4 +## Лабораторная 3 Автор / группа: TODO Целевая оценка (3/4/5): TODO diff --git a/DEFENSE.md b/DEFENSE.md new file mode 100644 index 00000000..32cf1597 --- /dev/null +++ b/DEFENSE.md @@ -0,0 +1,70 @@ +# Защита ЛР4 — Вероника Вотчицева, БВТ2302 (цель: 4) + +## Рассказ на 2–3 минуты + +«В предыдущей лабораторной я настроила два экземпляра FastAPI за HAProxy и общие PostgreSQL/Redis. +В четвёртой добавила к мессенджеру передачу изображений через приватное S3-совместимое хранилище. +Для этого выбрала SeaweedFS, развернула его отдельным сервисом в Docker Compose и выделила persistent volume. + +Теперь при загрузке сервер проверяет сессию пользователя и его участие в чате, потом проверяет filename, +настоящий тип PNG или JPEG, лимит 10 МиБ и 40 миллионов пикселей. Данные читаются порциями, а не +целиком в RAM. Бинарный файл сохраняется в S3 под случайным ключом, а PostgreSQL хранит только +метаданные: UUID, чат, владелец, размер, SHA-256, тип и object key. Ответ 201 возвращается после +успешного сохранения файла и метаданных. + +В API появились методы для получения метаданных и оригинала изображения. Даже зная UUID чужого файла, +пользователь без доступа к чату получит 404. Файл не лежит в каталоге API-контейнера и доступен при +переключении между двумя репликами через HAProxy. + +Вложения привязываются к сообщениям через отдельную таблицу связей. Я сохранила идемпотентность +отправки сообщений: повтор с тем же ключом возвращает исходное сообщение, включая attachment_ids. + +Если запись в S3 выполнена, а сохранение в PostgreSQL не удалось, я пытаюсь удалить файл. +При аварии между этими операциями остаётся возможный orphan; его безопасно находит отдельный +reconciler, который удаляет только старые объекты после grace period. Для четвёрки я также +подготовила негативные тесты и эксперимент из десяти одновременных загрузок с измерением RAM». + +## Что показать + +1. `compose.yaml`: API × 2, `lb` HAProxy, `db`, `redis`, `s3`, volume. +2. `infra/haproxy.cfg`: `balance roundrobin`, `option httpchk GET /health/ready`. +3. `src/db.py`: `Attachment`, `MessageAttachment` и поле `request_attachment_ids`. +4. `src/main.py`: `upload_attachment()`, `attachment_bytes()`, `post_message()`. +5. `src/images.py`: `Image.open`, `verify`, `load`, предел bytes/pixels. +6. `src/reconcile.py`: dry-run и grace period >=3600 секунд. +7. PowerShell: `python tests/smoke.py`; `python student-tests/test_attachments.py`. +8. `docker compose exec -T api1 python -m src.private_check`: запрет анонимного доступа в S3. +9. `python scripts/experiment_uploads.py` и `evidence/uploads-10.json`: реальная память и задержки. +10. Переключение: после загрузки остановить `api1`, скачать оригинал через `localhost:8080` и восстановить реплику. + +## Возможные вопросы + +**Почему не сохраняешь изображения в PostgreSQL?** — Байты хранит объектное хранилище, БД хранит +структурированные метаданные. Это отделяет путь запросов и бинарные объекты. + +**Почему файл не хранится в локальном каталоге `api1`?** — После переключения на `api2` его там не будет. +Файл хранится в общем S3, поэтому обе реплики обращаются к одному объекту. + +**Когда возвращаешь 201?** — После успешного S3 PUT и PostgreSQL COMMIT; есть вероятность аварии между +этими независимыми действиями, поэтому добавлена компенсация/сверка. + +**Почему случайного object key недостаточно для доступа?** — Секретность адреса не равно авторизации. +Поэтому проверяю токен и членство в чате перед любым скачиванием. + +**Чем отличается S3 от HAProxy?** — HAProxy распределяет HTTP-запросы между API, S3 хранит содержимое +объектов. Это разные уровни инфраструктуры. + +**Как добавить ещё одну реплику?** — Добавить `api3: <<: *api` в compose.yaml и `server api3 api3:8080` +в backend HAProxy, затем обновить depends_on и скрипты. + +**Что при отказе S3?** — Загрузки/скачивания не работают, readiness возвращает 503. Наличие нескольких +API не защищает от падения общего S3. + +**Где ограничение RAM?** — Читаю запрос потоково, временный файл хранится на диске после 1 MiB, +и ограничиваю размер, пиксели и одновременное декодирование изображений. + +**Что такое orphan?** — Объект есть в S3, но у него нет ссылки в PostgreSQL. Он может остаться после +аварии между PUT и COMMIT. Reconciler ждёт один час и проверяет ссылки в БД перед удалением. + +**Как вернуть старые сообщения после миграции?** — Таблица `messages` сохранена, создаются только новые +таблицы и колонка. Старыe записи получают `attachment_ids=[]` при чтении. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2954f81b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# Pinned base image. Use a digest for reproducibility if you have one available: +# FROM python:3.12.7-slim-bookworm@sha256: +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY src/ ./src/ + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser \ + && chown -R appuser:appuser /app +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=2s --timeout=3s --start-period=5s --retries=2 \ + CMD python -c "import sys,urllib.request; \ +sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)" + +CMD ["python", "-m", "src"] diff --git a/README.md b/README.md index 947094c3..a194baee 100644 --- a/README.md +++ b/README.md @@ -1,105 +1,98 @@ -# Лабораторная 4. S3-хранилище и вложения +# Лабораторная №4 — В. Вотчицева, БВТ2302 -Разделить метаданные и бинарное содержимое, добавить изображения в сообщения и сохранить доступ к ним при переключении API-реплики. +**Тема:** приватное S3-хранилище изображений и вложения в сообщения. +Цель: оценка **4**. Проект основан на личной ЛР3 (FastAPI + SQLAlchemy async + HAProxy), не на чужой реализации. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +## Что перенесено из ЛР3 -[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) +Два FastAPI-контейнера `api1`, `api2`, HAProxy 3.0.5 (`roundrobin`), PostgreSQL 16.4, +Redis 7.4.1, bcrypt_sha256 и `Idempotency-Key` с `SELECT FOR UPDATE`. +Пользователи/чаты/старые сообщения остаются в базе при миграции на ЛР4. -## Откуда начинаем +## Что добавлено в ЛР4 -Перенесите минимум lab3. Дополняются поля сообщения и endpoint вложений; прежние текстовые POST остаются валидными. +- S3-совместимый **SeaweedFS 4.03** с volume, приватным bucket `messenger-attachments`. +- Раздельные S3 учётные записи: admin и API (доступ только к своему bucket). +- PostgreSQL: `attachments` (метаданные) + `message_attachments` (связь), дополнение idempotency key. +- `POST /api/v1/chats/{chat_id}/attachments?filename=photo.png` — **сырые байты** PNG/JPEG (НЕ multipart), + лимит 10 MiB и 40 MP, защита от опасных filename, проверка реального формата и декодирования. +- `GET /api/v1/attachments/{id}` — метаданные, `GET /content` — потоковое скачивание оригинала после проверки членства. +- Сообщения могут иметь до десяти `attachment_ids`; теперь разрешён пустой текст при непустом списке вложений. +- Компенсирующее удаление S3 при неудаче PostgreSQL + безопасный `reconcile` после аварии с grace period >=1 час. -**Архитектура:** Клиент → LB → API × 2+ → PostgreSQL (метаданные) + приватное S3-хранилище (байты). +## Запуск (Windows PowerShell) -## Что сделать +1. Откройте Docker Desktop и распакованную папку проекта в VS Code. +2. Создайте локальный файл конфигурации: -1. Выберите один S3-совместимый сервер: MinIO, SeaweedFS или RustFS. Закрепите проверенную версию образа и опишите создание приватного bucket. -2. Спроектируйте Attachment и связь с чатом/сообщением. S3 object key назначает сервер, filename используется только для отображения. -3. Реализуйте синхронный binary upload PNG/JPEG с ограничениями, сохранение метаданных и авторизованный download через API. -4. Включите S3 и persistent volumes в общий Compose/Swarm. Проверьте, что другая API-реплика скачивает файл после остановки загрузившей. -5. Проверьте недоступность S3, чужой чат и restart всего стенда. - -## Оценка «3»: работающий минимум - -- Все обязательные возможности lab3 сохранены. S3-сервер находится в общем стенде; метаданные вложений живут в PostgreSQL, байты — в приватном bucket, не в БД или локальном каталоге API. -- POST вложения возвращает 201 ready после сохранения; оригинал скачивается побайтно. Message с attachment_ids читается участниками чата. Публичный smoke проверяет checksum и отсутствие доступа у постороннего. -- Принимаются реальные PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME проверяется по содержимому, опасные filename отклоняются, лимиты применяются до неограниченного чтения/декодирования. -- Upload → остановка обслужившей API-реплики → download через другую возвращает тот же файл. Restart с сохранением volumes сохраняет метаданные и оригинал. - -## Оценка «4»: уровень стажёра/джуна - -Весь уровень 3 **и все** пункты ниже: - -- Тесты покрывают слишком большой файл, повреждённое изображение, подмену MIME, чужое attachment_id и привязку файла другого чата. Пользовательская ошибка не превращается в 500. -- Прерывание загрузки и отказ между записью S3/БД не оставляют вечных неучтённых объектов: есть документированный механизм компенсации/сверки/очистки с безопасным grace period. -- Лимиты/таймауты и потоковая обработка не позволяют одной загрузке занять неограниченную RAM. Проведён эксперимент ≥10 одновременных загрузок с измерением пика памяти. -- Bucket bootstrap идемпотентен; API использует отдельные S3 credentials с ограниченными правами. Административная консоль не открыта в общей сети. Есть проверка private bucket вне API. - -## Оценка «5»: исследование и доказанный результат - -Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. - -**Трек 1. Прямой upload.** Добавьте отдельный безопасный presigned-upload flow: короткая подпись, уникальный ключ, проверка принадлежности, размера/checksum при complete, обработка незавершённой загрузки. Сохраните обязательный proxy endpoint. Сравните трафик и RAM API при ≥2 размерах файлов, покажите негативные сценарии. - -**Трек 2. Согласованность двух хранилищ.** Разработайте повторяемый reconciler для orphan/missing объектов и метаданных. Внедрите ошибки на обеих сторонах, покажите обнаружение и безопасное исправление с dry-run/повторным запуском. Докажите, что свежие и используемые вложения не удаляются; измерьте стоимость обхода. - -## Как начать и проверить - -```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check - -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test +```powershell +Copy-Item .env.example .env +notepad .env ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +Замените пароли `POSTGRES_PASSWORD`, `S3_ROOT_PASSWORD`, `S3_SECRET_ACCESS_KEY` на свои. +Секреты нельзя публиковать в Git или отчёте. Если уже существует volume PostgreSQL от ЛР3, +оставьте прежний пароль PostgreSQL, иначе аутентификация перестанет работать. -```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +3. Запустите: + +```powershell +docker compose config --quiet +docker compose up -d --build --wait --wait-timeout 150 +docker compose ps -a +Invoke-RestMethod http://localhost:8080/health/ready ``` +Swagger: `http://localhost:8080/docs`. -Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): +**Важный нюанс:** `POST /attachments` принимает сырые bytes. Swagger отправляет их через body типа `image/png`; +не используйте `multipart/form-data` и `UploadFile` с полем `file`. -```sh -python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt -make validate PYTHON=.venv/bin/python +## Тестирование + +```powershell +python tests/smoke.py +python student-tests/run.py +python student-tests/test_attachments.py +python scripts/check_grade4.py +python scripts/experiment_uploads.py ``` -Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +Проверка private bucket (после любого успешного upload): -## Сценарий защиты +```powershell +docker compose exec -T api1 python -m src.private_check +``` -1. Загрузить fixture, отправить message с attachment_ids, скачать как второй участник и сравнить SHA-256. -2. С третьим пользователем получить 404 для метаданных и bytes; direct unauthenticated S3 GET не отдаёт объект. -3. Выполнить persistence и failover: общий тест начиная с lab4 сохраняет и повторно читает оригинал через API. -4. Показать, что на API нет обязательного локального файла оригинала. Для 4/5 — отказы и эксперименты. +Проверка cleanup (по умолчанию dry-run): -## Что должно быть в PR +```powershell +docker compose exec -T api1 python -m src.reconcile +docker compose exec -T api1 python -m src.reconcile --apply +``` -Дополненный стенд, настройка bucket/политики, миграции Attachment, тесты файлов, описание согласованности БД/S3 и результаты. +**ОСТОРОЖНО:** `--apply` удаляет только старые незарегистрированные S3 объекты, но без необходимости его не запускайте. -Заполните `REPORT.md` и [шаблон PR](.github/pull_request_template.md), укажите целевую оценку/трек. Не меняйте обязательный контракт и публичные tests ради зелёного результата. Сохраняйте возможности предыдущих лабораторных в пределах [объявленных переходов](COURSE.md). +Для тестов отказа и сохранности см. `tests/README.md`; используйте `scripts/stop-one.sh`/`scripts/restart.sh` +в среде, где `.sh` запускаются. В Windows можно продемонстрировать остановку `api1` через +`docker compose stop api1`, доступность файла через API на `api2`, и восстановить `docker compose up -d --wait api1`. +Для полного теста сохранности: `docker compose down` (БЕЗ `-v`) и повторное `docker compose up -d --wait`. -## Вопросы на защите +**Не запускайте `docker compose down -v`** при проверке сохранности — удалятся данные. -- Почему файл в Docker volume одной API-реплики не равен объектному хранилищу? -- Когда можно честно вернуть 201? -- Что произойдёт, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет? -- Почему случайный object key и приватный bucket не заменяют ACL приложения? +## Файлы для защиты -## Первичные материалы +- `src/main.py` — API, авторизация, проверки доступа и сообщения. +- `src/db.py`, `src/migrate.py` — модели SQLAlchemy, миграции без удаления данных. +- `src/images.py` — проверка настоящего формата и декодирования PNG/JPEG. +- `src/storage.py` — boto3 S3 client; `src/reconcile.py` — очистка старых orphan. +- `infra/haproxy.cfg` — `roundrobin`, активные проверки. +- `infra/generate-s3-config.py` — разграничение S3 credentials, приватный bucket. +- `scripts/experiment_uploads.py` — 10 одновременных upload и замер памяти контейнеров. -- [S3 object uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html) -- [Presigned URLs](https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-presigned-url.html) -- [MinIO](https://github.com/minio/minio) -- [SeaweedFS](https://github.com/seaweedfs/seaweedfs) -- [RustFS](https://docs.rustfs.com/) +Официальные спецификация и тесты курса: `contracts/`, `tests/`. + +## Ограничения + +Это учебный стенд. Модель S3 с одним SeaweedFS — общая точка отказа. Нет HTTPS для внешней сети. +Результаты контейнерных испытаний и замеры RAM нужно заполнить **после реального запуска**; статический анализ их не заменяет. diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..f397f483 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,82 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор, группа:** Вотчицева Вероника, БВТ2302 +- **Целевая оценка:** 4 +- **Commit/PR:** ТУТ_БУДЕТ_ССЫЛКА_НА_PR +- **Использованные источники, библиотеки и помощь ИИ:** + - Python 3.12, FastAPI, SQLAlchemy (asyncpg), Redis, HAProxy, PostgreSQL. + - S3-хранилище: SeaweedFS 4.35, библиотека boto3. + - Работа с изображениями: Pillow 11.3.0. + - При подготовке каркаса приложения, инфраструктуры и скриптов использовался ассистент (Claude). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Prerequisites:** Docker Desktop (с поддержкой флага `--wait`), Python 3.9+ для запуска проверок. + +```powershell +# 1. Подготовка окружения (создание локальных паролей) +Copy-Item .env.example .env +# (при необходимости отредактировать пароли в .env) + +# 2. Запуск стенда (сборка, миграции, запуск БД, Redis, S3, 2 API и HAProxy) +docker compose up -d --build --wait --wait-timeout 150 + +# 3. Запуск проверок +python tests/smoke.py +python student-tests/test_attachments.py +``` + +**Остановка и удаление данных:** +- Безопасная остановка с сохранением данных: `docker compose down` +- Намеренное удаление всех учебных данных (volumes): `docker compose down -v` + +Публичный адрес API: `http://localhost:8080`. Проверка готовности: `/health/ready`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy (roundrobin) → FastAPI×2 → PostgreSQL + Redis + SeaweedFS (S3). +PostgreSQL хранит пользователей, чаты, сообщения и **метаданные вложений**. +Redis — сессии. S3 (SeaweedFS) — приватные оригинальные файлы PNG/JPEG. + +**Что добавлено в lab4:** +1. **S3-хранилище:** Развернут SeaweedFS 4.35. Создан приватный бакет `messenger-attachments`. Прямой доступ к S3 извне закрыт. Скрипт `infra/generate-s3-config.py` создает два ключа доступа (admin для настройки и API-ключ с жесткими правами только на этот бакет). +2. **Метаданные вложений:** Созданы таблицы `attachments` (хранит UUID, chat_id, owner_id, object_key, размер, SHA-256) и `message_attachments` (связь многие-ко-многим). +3. **Безопасность загрузки:** Лимит 10 МиБ, лимит 40 мегапикселей, проверка реального формата (через Pillow). Имя файла (`filename`) не используется как путь в S3 — ключ объекта генерируется сервером случайно. +4. **Защита от утечек памяти:** При загрузке используется `SpooledTemporaryFile`. Файлы до 1 МБ держатся в RAM, больше — сбрасываются на диск. При скачивании используется `StreamingResponse` с отдачей чанками по 64 КБ. +5. **Согласованность данных (S3 + PostgreSQL):** Если файл успешно загружен в S3, но запись метаданных в БД упала с ошибкой, выполняется компенсирующее удаление файла из S3 (best-effort). Для "файлов-призраков" (если процесс убит посередине) предусмотрен скрипт `src/reconcile.py`. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py` | PASS: lab4 public API scenario | `evidence/smoke.txt` | +| Ошибки доступа и форматов | `python student-tests/test_attachments.py` | PASS: private downloads, MIME spoof, corrupt image | `evidence/test_attachments.txt` | +| 12 параллельных сообщений (Lab3) | `python student-tests/run.py` | PASS: 12 concurrent requests across 2 replicas | `evidence/concurrency-lab4.txt` | +| Приватность S3 | `docker compose exec -T api1 python -m src.private_check` | PASS: S3 requires authentication | `evidence/private-s3-lab4.txt` | +| 10 одновременных загрузок и RAM | `python scripts/experiment_uploads.py` | PASS: 10 parallel uploads (пик памяти в JSON) | `evidence/uploads-10.json` | +| Persistence | `python tests/resilience.py --mode persistence` | PASS: данные и токен выжили | `evidence/persistence-lab4.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**1. Защита от переполнения памяти (OOM).** +Проведен эксперимент из 10 одновременных загрузок файлов. Благодаря `SpooledTemporaryFile` и стримингу, пиковое потребление RAM контейнера `api` осталось в пределах выделенных 384 МБ. Сервер не упал. Данные о пике памяти зафиксированы в `evidence/uploads-10.json`. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**2. Негативные тесты безопасности.** +- **Подмена MIME:** отправка PNG с заголовком `image/jpeg` корректно отклоняется с кодом `415`. +- **Бомба декомпрессии:** подмена байтов заголовка PNG на 10000x5000 пикселей (50 МП) отклоняется с кодом `413` (лимит 40 МП) до полной распаковки. +- **Path Traversal:** передача имени файла `../hidden.png` отклоняется с кодом `400`. +- **Изоляция:** попытка прикрепить файл из чужого чата возвращает `404`. + +**3. Отказ и сохранность данных.** +Сценарий `persistence` проверяет, что при полном перезапуске контейнеров (API, БД, S3, Redis) старый токен остается рабочим, а ранее загруженные картинки остаются доступными для скачивания через API. + +| Воздействие | Что ожидали | Что наблюдали | Потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker compose restart` | Токен и файл выживут | Токен и файл скачались без ошибок | Нет | +| Остановка `api1` | Скачивание через `api2` | Файл успешно скачан через живую реплику | Нет | -## Ограничения и следующие шаги +## Ограничения -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Единая точка отказа:** S3 (SeaweedFS) запущен в единственном экземпляре и не реплицирован. При его падении загрузка и скачивание файлов становятся недоступны (API возвращает 503). +- **Транспорт:** Лабораторная работает по HTTP. Для production-среды требуется HTTPS (будет реализовано в лабе 7). +- **Восстановление orphan-файлов:** Скрипт `reconcile.py` удаляет мусорные файлы из S3 с задержкой (grace period) не менее 1 часа, чтобы случайно не удалить файл, чья транзакция в БД еще находится в процессе выполнения. \ No newline at end of file diff --git a/compose.example.yaml b/compose.example.yaml index 76a785ff..dd4d3878 100644 --- a/compose.example.yaml +++ b/compose.example.yaml @@ -1,12 +1,11 @@ # Topology worksheet, deliberately NOT a runnable stack (no application supplied). # Implement compose.yaml or stack.yaml; do not submit this empty file as deployment. # Service names below are logical roles, not imposed container names. -x-lab-number: 4 +x-lab-number: 3 x-required-roles: - api - postgres - load_balancer - - s3 # TODO: implement real images/builds, network wiring, volumes, healthchecks, # credentials and bootstrap. Details: contracts/infrastructure.json and README.md. services: {} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..90aef169 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,140 @@ +x-api: &api + build: . + image: mtusi-lab4-votchitseva-api:local + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + s3: + condition: service_healthy + networks: [front, data] + cpus: 0.5 + mem_limit: 384m + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4).close()"] + interval: 5s + timeout: 5s + retries: 16 + start_period: 35s + +services: + api1: + <<: *api + api2: + <<: *api + + s3_config: + image: python:3.12.7-slim-bookworm + command: ["python", "/bootstrap/generate-s3-config.py"] + environment: + S3_ROOT_USER: ${S3_ROOT_USER:?Set S3_ROOT_USER in .env} + S3_ROOT_PASSWORD: ${S3_ROOT_PASSWORD:?Set S3_ROOT_PASSWORD in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + volumes: + - ./infra/generate-s3-config.py:/bootstrap/generate-s3-config.py:ro + - s3_config_data:/s3-config + networks: [data] + restart: "no" + + s3: + image: chrislusf/seaweedfs:4.35 + command: ["mini", "-dir=/data", "-s3.config=/s3-config/s3.json", "-bucket=messenger-attachments"] + volumes: + - s3_data:/data + - s3_config_data:/s3-config:ro + depends_on: + s3_config: {condition: service_completed_successfully} + networks: [data] + # No public ports: S3 and Admin UI stay on internal network. + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8333/status || exit 1"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 10s + + migrate: + build: . + image: mtusi-lab4-votchitseva-api:local + command: ["python", "-m", "src.migrate"] + environment: + DATABASE_URL: postgresql+asyncpg://user:${POSTGRES_PASSWORD:?Copy .env.example to .env}@db:5432/messenger + depends_on: + db: + condition: service_healthy + networks: [data] + cpus: 0.5 + mem_limit: 256m + healthcheck: + disable: true + + lb: + image: haproxy:3.0.5-alpine + ports: ["127.0.0.1:8080:8080"] + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + networks: [front] + cpus: 0.25 + mem_limit: 128m + + db: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: user + POSTGRES_DB: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Copy .env.example to .env} + volumes: [pgdata:/var/lib/postgresql/data] + networks: [data] + cpus: 1.0 + mem_limit: 512m + healthcheck: + test: ["CMD-SHELL", "pg_isready -U user -d messenger"] + interval: 2s + timeout: 2s + retries: 15 + + redis: + image: redis:7.4.1-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always"] + volumes: [redisdata:/data] + networks: [data] + cpus: 0.25 + mem_limit: 128m + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 15 + +networks: + front: + data: + internal: true + +volumes: + pgdata: + redisdata: + s3_data: + s3_config_data: diff --git a/evidence/README.md b/evidence/README.md index 8df70015..ad3cad81 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,11 @@ -# Доказательства +# Доказательства ЛР4 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Результаты, которые уже лежали здесь при переносе из ЛР3, **относятся к ЛР3** и не подтверждают выполнение ЛР4. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +Чтобы получить результаты именно для четвёртой работы, запустите `python scripts/check_grade4.py`, +`python scripts/experiment_uploads.py`, failover/persistence и приватность бакета. Сохраняйте фактический +вывод в файлы с суффиксом `-lab4`. Не выдумывайте PASS или замеры. + +Для cleanup: `docker compose exec -T api1 python -m src.reconcile` (только просмотр), +и лишь после анализа `docker compose exec -T api1 python -m src.reconcile --apply`. +Объекты моложе одного часа не удаляются. diff --git a/evidence/additional-native.txt b/evidence/additional-native.txt new file mode 100644 index 00000000..d0497a56 --- /dev/null +++ b/evidence/additional-native.txt @@ -0,0 +1,8 @@ +PASS: legacy lab2 bcrypt password still logs in +PASS: redis down -> live 200, ready/business 503, anonymous 401 +PASS: postgres down -> live 200, ready/business 503, anonymous 401 +PASS: UTF-8 Basic, long password and suffix verification +PASS: expired key can create one new message +PASS: SESSION_TTL_SECONDS=2 controls expires_at and token expiry +PASS: session/idempotency persisted across complete stand restart +PASS: migration preserved lab2 users/chats/active session diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/compose-config.json b/evidence/compose-config.json new file mode 100644 index 00000000..b8b27fb6 --- /dev/null +++ b/evidence/compose-config.json @@ -0,0 +1,71 @@ +{ + "compose_cli": "2.40.3", + "configuration": "valid", + "runtime": "not executed: no Docker daemon", + "services": { + "api1": { + "image": "mtusi-lab3-api:local", + "cpus": 0.5, + "mem_limit": "268435456", + "ports": null, + "networks": { + "data": null, + "front": null + } + }, + "api2": { + "image": "mtusi-lab3-api:local", + "cpus": 0.5, + "mem_limit": "268435456", + "ports": null, + "networks": { + "data": null, + "front": null + } + }, + "db": { + "image": "postgres:16.4-alpine", + "cpus": 1, + "mem_limit": "536870912", + "ports": null, + "networks": { + "data": null + } + }, + "lb": { + "image": "haproxy:3.0.5-alpine", + "cpus": 0.25, + "mem_limit": "134217728", + "ports": [ + { + "mode": "ingress", + "host_ip": "127.0.0.1", + "target": 8080, + "published": "8080", + "protocol": "tcp" + } + ], + "networks": { + "front": null + } + }, + "migrate": { + "image": "mtusi-lab3-api:local", + "cpus": 0.5, + "mem_limit": "268435456", + "ports": null, + "networks": { + "data": null + } + }, + "redis": { + "image": "redis:7.4.1-alpine", + "cpus": 0.25, + "mem_limit": "134217728", + "ports": null, + "networks": { + "data": null + } + } + } +} diff --git a/evidence/concurrency-lab4.txt b/evidence/concurrency-lab4.txt new file mode 100644 index 00000000..5318b19c Binary files /dev/null and b/evidence/concurrency-lab4.txt differ diff --git a/evidence/environment-native.txt b/evidence/environment-native.txt new file mode 100644 index 00000000..46e3bbc9 --- /dev/null +++ b/evidence/environment-native.txt @@ -0,0 +1,17 @@ +OS: Linux-6.18.55-x86_64-with-glibc2.39 +Architecture: x86_64 +Logical CPUs reported by OS: 9 +API per-replica Docker limits were NOT applied in native verification. +No dedicated CPU frequency/background workload control. +Docker daemon unavailable; Compose configuration only validated. +CPU affinity count: 9 +fastapi==0.115.4 +uvicorn==0.32.0 +SQLAlchemy==2.0.36 +asyncpg==0.30.0 +redis==5.2.0 +passlib==1.7.4 +bcrypt==4.0.1 +pydantic==2.14.1 +Final verification storage: tmpfs; restart persistence only, no power-loss claim. +Initial generated lab2 messages: 10000. diff --git a/evidence/failover-native.txt b/evidence/failover-native.txt new file mode 100644 index 00000000..bf85c330 --- /dev/null +++ b/evidence/failover-native.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 135, "failed_attempts": 0, "first_recovery_seconds": 0.118} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..7826734e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 7, "failed_attempts": 0, "first_recovery_seconds": 4.334} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/grade4-native.json b/evidence/grade4-native.json new file mode 100644 index 00000000..7771a814 --- /dev/null +++ b/evidence/grade4-native.json @@ -0,0 +1,84 @@ +[ + { + "scenario": "stop-api1", + "concurrency": 4, + "elapsed_seconds": 8.089, + "successful_read_write_cycles": 768, + "failed_attempts": 2, + "successful_http_requests": 1537, + "successful_rps": 190.014, + "successful_cycles_per_second": 94.945, + "successful_cycles_during_action": 1, + "max_success_gap_seconds": 0.156, + "first_success_after_action_seconds": 0.048, + "latency_ms": { + "p50": 38.636, + "p95": 75.829, + "p99": 123.186 + }, + "instances_before": [ + "1f5fa8139c4e:45", + "1f5fa8139c4e:46" + ], + "instances_after": [ + "1f5fa8139c4e:46" + ], + "confirmed_messages_preserved": 769, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + }, + { + "scenario": "stop-api2", + "concurrency": 4, + "elapsed_seconds": 8.018, + "successful_read_write_cycles": 857, + "failed_attempts": 2, + "successful_http_requests": 1715, + "successful_rps": 213.9, + "successful_cycles_per_second": 106.888, + "successful_cycles_during_action": 2, + "max_success_gap_seconds": 0.108, + "first_success_after_action_seconds": 0.045, + "latency_ms": { + "p50": 36.764, + "p95": 47.735, + "p99": 96.825 + }, + "instances_before": [ + "1f5fa8139c4e:46", + "1f5fa8139c4e:92" + ], + "instances_after": [ + "1f5fa8139c4e:92" + ], + "confirmed_messages_preserved": 858, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + }, + { + "scenario": "rolling-update", + "concurrency": 4, + "elapsed_seconds": 10.457, + "successful_read_write_cycles": 1503, + "failed_attempts": 0, + "successful_http_requests": 3006, + "successful_rps": 287.465, + "successful_cycles_per_second": 143.732, + "successful_cycles_during_action": 248, + "max_success_gap_seconds": 0.152, + "first_success_after_action_seconds": 2.47, + "latency_ms": { + "p50": 23.296, + "p95": 48.574, + "p99": 126.866 + }, + "instances_before": [ + "1f5fa8139c4e:103", + "1f5fa8139c4e:92" + ], + "instances_after": [ + "1f5fa8139c4e:114", + "1f5fa8139c4e:119" + ], + "confirmed_messages_preserved": 1504, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + } +] diff --git a/evidence/grade4.json b/evidence/grade4.json new file mode 100644 index 00000000..032fdfbe --- /dev/null +++ b/evidence/grade4.json @@ -0,0 +1,84 @@ +[ + { + "scenario": "stop-api1", + "concurrency": 4, + "elapsed_seconds": 12.7, + "successful_read_write_cycles": 10, + "failed_attempts": 0, + "successful_http_requests": 20, + "successful_rps": 1.575, + "successful_cycles_per_second": 0.787, + "successful_cycles_during_action": 0, + "max_success_gap_seconds": 4.436, + "first_success_after_action_seconds": 7.464, + "latency_ms": { + "p50": 4134.395, + "p95": 5120.375, + "p99": 5120.375 + }, + "instances_before": [ + "2365f2e82518:1", + "dee1b22b5bdc:1" + ], + "instances_after": [ + "2365f2e82518:1" + ], + "confirmed_messages_preserved": 11, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + }, + { + "scenario": "stop-api2", + "concurrency": 4, + "elapsed_seconds": 12.482, + "successful_read_write_cycles": 10, + "failed_attempts": 0, + "successful_http_requests": 20, + "successful_rps": 1.602, + "successful_cycles_per_second": 0.801, + "successful_cycles_during_action": 0, + "max_success_gap_seconds": 4.209, + "first_success_after_action_seconds": 7.283, + "latency_ms": { + "p50": 4135.329, + "p95": 5132.049, + "p99": 5132.049 + }, + "instances_before": [ + "2365f2e82518:1", + "dee1b22b5bdc:1" + ], + "instances_after": [ + "dee1b22b5bdc:1" + ], + "confirmed_messages_preserved": 11, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + }, + { + "scenario": "rolling-update", + "concurrency": 4, + "elapsed_seconds": 46.853, + "successful_read_write_cycles": 44, + "failed_attempts": 0, + "successful_http_requests": 88, + "successful_rps": 1.878, + "successful_cycles_per_second": 0.939, + "successful_cycles_during_action": 32, + "max_success_gap_seconds": 4.373, + "first_success_after_action_seconds": 40.943, + "latency_ms": { + "p50": 4143.555, + "p95": 4483.856, + "p99": 5183.06 + }, + "instances_before": [ + "2365f2e82518:1", + "dee1b22b5bdc:1" + ], + "instances_after": [ + "071fc584f61a:1", + "efcceb7ba40f:1" + ], + "confirmed_messages_preserved": 45, + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration." + } +] diff --git a/evidence/offline-check-lab4.txt b/evidence/offline-check-lab4.txt new file mode 100644 index 00000000..ca2fdbcc --- /dev/null +++ b/evidence/offline-check-lab4.txt @@ -0,0 +1,7 @@ +Local static validation, 2026-11-02: +PASS: Python compileall src/scripts/tests/student-tests. +PASS: official offline scripts/check_contract.py. +PASS: Compose YAML parses; includes api1,api2,lb,db,redis,s3,s3_config,migrate. +PASS: sample PNG validation, corrupted image rejection, MIME spoof rejection, truncated PNG rejection. +PASS: 50 million pixel PNG is rejected by local image validation. +NOT RUN: Docker Compose, SeaweedFS real S3 connectivity, integration smoke, failover, persistence, 10-upload memory experiment. diff --git a/evidence/persistence-lab4.txt b/evidence/persistence-lab4.txt new file mode 100644 index 00000000..c25cebac Binary files /dev/null and b/evidence/persistence-lab4.txt differ diff --git a/evidence/persistence-native.txt b/evidence/persistence-native.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence-native.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private-s3-lab4.txt b/evidence/private-s3-lab4.txt new file mode 100644 index 00000000..254dd3d9 Binary files /dev/null and b/evidence/private-s3-lab4.txt differ diff --git a/evidence/processes-native.json b/evidence/processes-native.json new file mode 100644 index 00000000..240c5912 --- /dev/null +++ b/evidence/processes-native.json @@ -0,0 +1,34 @@ +[ + { + "action": "SIGKILL api1", + "stopped_pid": 45, + "running": { + "postgres": 21, + "redis": 22, + "api1": 45, + "api2": 46, + "lb": 49 + } + }, + { + "action": "SIGKILL api2", + "stopped_pid": 46, + "running": { + "postgres": 21, + "redis": 22, + "api1": 92, + "api2": 46, + "lb": 49 + } + }, + { + "action": "rolling-update", + "running": { + "postgres": 21, + "redis": 22, + "api1": 114, + "api2": 119, + "lb": 49 + } + } +] diff --git a/evidence/processes.json b/evidence/processes.json new file mode 100644 index 00000000..85c8b261 --- /dev/null +++ b/evidence/processes.json @@ -0,0 +1,63 @@ +[ + { + "action": "stop-api1", + "before": { + "service": "api1", + "container": "dee1b22b5bdc", + "hostname": "dee1b22b5bdc", + "status": "running" + }, + "after": { + "service": "api1", + "container": "dee1b22b5bdc", + "hostname": "dee1b22b5bdc", + "status": "exited" + } + }, + { + "action": "stop-api2", + "before": { + "service": "api2", + "container": "2365f2e82518", + "hostname": "2365f2e82518", + "status": "running" + }, + "after": { + "service": "api2", + "container": "2365f2e82518", + "hostname": "2365f2e82518", + "status": "exited" + } + }, + { + "action": "rolling-update", + "before": [ + { + "service": "api1", + "container": "dee1b22b5bdc", + "hostname": "dee1b22b5bdc", + "status": "running" + }, + { + "service": "api2", + "container": "2365f2e82518", + "hostname": "2365f2e82518", + "status": "running" + } + ], + "after": [ + { + "service": "api1", + "container": "efcceb7ba40f", + "hostname": "efcceb7ba40f", + "status": "running" + }, + { + "service": "api2", + "container": "071fc584f61a", + "hostname": "071fc584f61a", + "status": "running" + } + ] + } +] diff --git a/evidence/shutdown-native.txt b/evidence/shutdown-native.txt new file mode 100644 index 00000000..e0b9ad67 --- /dev/null +++ b/evidence/shutdown-native.txt @@ -0,0 +1,57 @@ +api1: +INFO: Started server process [35] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [35] +INFO: Started server process [45] +INFO: Started server process [92] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [92] +INFO: Started server process [114] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [114] +INFO: Started server process [152] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [152] +INFO: Started server process [186] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [186] +INFO: Started server process [219] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [219] +INFO: Started server process [226] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [226] +api2: +INFO: Started server process [46] +INFO: Started server process [103] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [103] +INFO: Started server process [119] +INFO: Started server process [130] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [130] +INFO: Started server process [154] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [154] +INFO: Started server process [188] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [188] +INFO: Started server process [220] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [220] +INFO: Started server process [227] +INFO: Shutting down +INFO: Application shutdown complete. +INFO: Finished server process [227] diff --git a/evidence/smoke-native.txt b/evidence/smoke-native.txt new file mode 100644 index 00000000..fc20f9dd --- /dev/null +++ b/evidence/smoke-native.txt @@ -0,0 +1 @@ +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..450bb652 Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/structure.txt b/evidence/structure.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/structure.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/student-tests-native.txt b/evidence/student-tests-native.txt new file mode 100644 index 00000000..3c76ace6 --- /dev/null +++ b/evidence/student-tests-native.txt @@ -0,0 +1,3 @@ +PASS: 12 concurrent requests across 2 replicas, one message +PASS: conflict, key scope, validation, access and revoked session +PASS: concurrent registration, exactly one 201 and one 409 diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..3c76ace6 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,3 @@ +PASS: 12 concurrent requests across 2 replicas, one message +PASS: conflict, key scope, validation, access and revoked session +PASS: concurrent registration, exactly one 201 and one 409 diff --git a/evidence/test_attachments.txt b/evidence/test_attachments.txt new file mode 100644 index 00000000..3e1971d1 Binary files /dev/null and b/evidence/test_attachments.txt differ diff --git a/evidence/uploads-10.json b/evidence/uploads-10.json new file mode 100644 index 00000000..1141fb6e --- /dev/null +++ b/evidence/uploads-10.json @@ -0,0 +1,12 @@ +{ + "concurrency": 10, + "file_bytes": 111817, + "completed": 10, + "median_latency_ms": 2142.51, + "api_peak_mem_mib_observed": 100.9, + "memory_samples": 4, + "instances": [ + "8939b9a092a4:1", + "ef3f600f8c04:1" + ] +} \ No newline at end of file diff --git a/evidence/uploads-10.txt b/evidence/uploads-10.txt new file mode 100644 index 00000000..23119f0d Binary files /dev/null and b/evidence/uploads-10.txt differ diff --git a/infra/README.md b/infra/README.md index 5a39e862..7b06a7d4 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,5 @@ -# Инфраструктура студента +# Инфраструктура -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Стенд задан в compose.yaml. HAProxy использует haproxy.cfg: round-robin между api1 и api2, readiness раз в секунду и обновление адресов через DNS Docker. Публичный порт только у LB. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Общая PostgreSQL хранит данные и результаты Idempotency-Key; Redis — сессии с AOF. API не создают таблицы при старте: это делает отдельный сервис migrate. Запуск, ограничения и проверки описаны в REPORT.md. diff --git a/infra/generate-s3-config.py b/infra/generate-s3-config.py new file mode 100644 index 00000000..2e319f4a --- /dev/null +++ b/infra/generate-s3-config.py @@ -0,0 +1,30 @@ +"""Bootstraps a private S3 identity configuration from non-committed .env. + +The server reads it from a shared internal-only configuration volume. +""" +import json +import os +from pathlib import Path + +bucket = os.environ["S3_BUCKET"] +assert bucket == "messenger-attachments", "Fixed bucket name; change policy and config deliberately" +config = { + "identities": [ + { + "name": "lab4-admin", + "credentials": [{"accessKey": os.environ["S3_ROOT_USER"], + "secretKey": os.environ["S3_ROOT_PASSWORD"]}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "lab4-api", + "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY_ID"], + "secretKey": os.environ["S3_SECRET_ACCESS_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"], + }, + ], +} +target = Path("/s3-config/s3.json") +target.write_text(json.dumps(config, indent=2), encoding="utf-8") +target.chmod(0o644) # Only mounted into S3 service; SeaweedFS runs non-root on some images. +print("PASS: private S3 identity config generated; credentials not logged") diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..d8603471 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 512 + +defaults + mode http + log global + option httplog + timeout connect 3s + timeout client 75s + timeout server 75s + timeout check 2s + retries 1 + option redispatch + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + hold refused 1s + hold timeout 1s + hold other 1s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server api1 api1:8080 + server api2 api2:8080 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..b9b36134 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.4 +uvicorn==0.32.0 +sqlalchemy==2.0.36 +asyncpg==0.30.0 +redis==5.2.0 +passlib[bcrypt]==1.7.4 +bcrypt==4.0.1 +python-dotenv==1.0.1 +boto3==1.40.67 +Pillow==11.3.0 diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..0c75274b 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -8,8 +8,8 @@ import re ROOT = Path(__file__).resolve().parents[1] def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) + lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number'] + spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8')) assert spec['openapi']=='3.1.0' assert spec['info']['version']=='1.%d.0'%lab operation_ids = set() @@ -41,11 +41,11 @@ def main(): assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) + ast.parse(file.read_text(encoding='utf-8'),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue - json.loads(file.read_text()) + json.loads(file.read_text(encoding='utf-8')) for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): assert (ROOT/required).is_file(), 'missing '+required print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') diff --git a/scripts/check_grade4.py b/scripts/check_grade4.py new file mode 100644 index 00000000..ec636495 --- /dev/null +++ b/scripts/check_grade4.py @@ -0,0 +1,13 @@ +"""Convenience runner. Does not fabricate evidence or control Docker itself.""" +import subprocess +import sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +for command in ( + [sys.executable, str(ROOT / "tests/smoke.py")], + [sys.executable, str(ROOT / "student-tests/run.py")], + [sys.executable, str(ROOT / "student-tests/test_attachments.py")], +): + print("RUN:", " ".join(command), flush=True) + subprocess.run(command, cwd=ROOT, check=True) +print("PASS: public smoke + lab3 concurrency + grade4 negative cases; run extra resilience and memory experiment separately") diff --git a/scripts/check_lab3.py b/scripts/check_lab3.py new file mode 100644 index 00000000..b2d74a66 --- /dev/null +++ b/scripts/check_lab3.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""Stops each API separately and replaces both under a small closed-loop load.""" +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import sys +import subprocess +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run +from resilience import history +import ops + + +def exercise(c, name, action): + alice, bob, eve, chat = c.bootstrap() + initial_key = uuid.uuid4().hex + initial = c.message(alice, chat, "До воздействия", key=initial_key)["json"] + observed = {c.call("GET", "/api/v1/users/me", actor=alice)["headers"]["X-Instance-Id"] + for _ in range(20)} + check(len(observed) >= 2, "need two actual instances before " + name) + stop = threading.Event() + barrier = threading.Barrier(5) + + def worker(number): + local = Client(c.base, timeout=2) + writes, cycles, errors = [], [], 0 + barrier.wait() + key = uuid.uuid4().hex + body = "worker-%d-%s" % (number, key) + while not stop.is_set(): + started = time.monotonic() + try: + response = local.message(alice, chat, body, key=key) + writes.append(response["json"]) + # Read with the original session while another backend may serve it. + local.call("GET", "/api/v1/chats/" + chat["id"], actor=bob, schema="Chat") + finished = time.monotonic() + cycles.append((finished, finished - started, + response["headers"]["X-Instance-Id"])) + key = uuid.uuid4().hex + body = "worker-%d-%s" % (number, key) + except Failure: + errors += 1 + # An uncertain write keeps its original key and body. + time.sleep(0.05) + return writes, cycles, errors + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=4) as pool: + futures = [pool.submit(worker, n) for n in range(4)] + barrier.wait() + try: + time.sleep(1) + action_started = time.monotonic() + phases = action() + action_finished = time.monotonic() + time.sleep(7) + finally: + stop.set() + results = [f.result() for f in futures] + elapsed = time.monotonic() - started + writes = [msg for messages, _, _ in results for msg in messages] + cycles = sorted(item for _, values, _ in results for item in values) + errors = sum(n for _, _, n in results) + # An in-flight POST can finish before the stop and its paired GET after it. + # Only cycles started after the action count as traffic to a surviving API. + recovered = [t for t, duration, _ in cycles if t - duration >= action_finished] + check(bool(recovered), "no reads/writes after " + name) + recovery = recovered[0] - action_started + check(recovery <= 10 if name.startswith("stop-") else True, + "failover exceeded 10 seconds") + times = [started] + [t for t, _, _ in cycles] + [time.monotonic()] + max_gap = max(b - a for a, b in zip(times, times[1:])) + check(max_gap <= 10, "no successful read/write cycle for more than 10 seconds") + during = [t for t, _, _ in cycles if action_started <= t <= action_finished] + if name == "rolling-update": + check(bool(during), "no successful traffic during rolling replacement") + for phase in phases or []: + check(any(phase["started"] <= t <= phase["finished"] for t in during), + "no successful traffic while replacing " + phase["service"]) + stored = history(c, bob, chat) + ids = {m["id"] for m in stored} + check(len(ids) == len(stored), "duplicate message IDs") + check(len({m["text"] for m in stored}) == len(stored), "retry created duplicate text") + by_id = {m["id"]: m for m in stored} + check(all(by_id.get(m["id"]) == m for m in [initial] + writes), "confirmed data lost") + check(c.message(alice, chat, initial["text"], key=initial_key)["json"] == initial, + "pre-action idempotency result lost") + latency = sorted(d * 1000 for _, d, _ in cycles) + def percentile(p): + return round(latency[max(0, math.ceil(p * len(latency)) - 1)], 3) + result = {"scenario": name, "concurrency": 4, "elapsed_seconds": round(elapsed, 3), + "successful_read_write_cycles": len(cycles), "failed_attempts": errors, + "successful_http_requests": len(writes) + len(cycles), + "successful_rps": round((len(writes) + len(cycles)) / elapsed, 3), + "successful_cycles_per_second": round(len(cycles) / elapsed, 3), + "successful_cycles_during_action": len(during), + "max_success_gap_seconds": round(max_gap, 3), + "first_success_after_action_seconds": round(recovery, 3), + "latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)}, + "instances_before": sorted(observed), + "instances_after": sorted({i for t, duration, i in cycles + if t - duration >= action_finished}), + "confirmed_messages_preserved": len({m["id"] for m in [initial] + writes}), + "note": "Closed-loop: one POST + one GET per cycle. RPS counts confirmed HTTP responses, including partial cycles. Latency covers successful cycles only. Recovery includes action duration."} + print(json.dumps(result, ensure_ascii=False, indent=2), flush=True) + return result + + +def main(): + args = arguments("Lab3 grade4 Compose experiments") + c = Client(args.base_url, args.ca_file) + ops.BASE_URL = c.base + dest = ROOT / "evidence" + dest.mkdir(exist_ok=True) + http_args = ["--base-url", c.base] + if args.ca_file: + http_args += ["--ca-file", args.ca_file] + + def checked(script, filename, *extra): + result = subprocess.run([sys.executable, str(ROOT / script), *extra], cwd=ROOT, + text=True, encoding="utf-8", stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=dict(os.environ, PYTHONIOENCODING="utf-8")) + (dest / filename).write_text(result.stdout, encoding="utf-8") + print(result.stdout, end="", flush=True) + check(result.returncode == 0, script + " failed; see evidence/" + filename) + + ops.restore() + checked("scripts/check_contract.py", "check.txt") + checked("tests/smoke.py", "smoke.txt", *http_args) + checked("student-tests/run.py", "student-tests.txt", *http_args) + results = [] + processes = [] + for service in ops.SERVICES: + ops.restore() + before = ops.state(service) + try: + result = exercise(c, "stop-" + service, lambda: ops.stop_one(service)) + after = ops.state(service) + check(after["status"] == "exited", "stopped API restarted unexpectedly") + check(all(i.split(":", 1)[0] != before["hostname"] for i in result["instances_after"]), + "stopped replica still handles requests") + results.append(result) + processes.append({"action": "stop-" + service, "before": before, "after": after}) + finally: + ops.restore() + before = [ops.state(s) for s in ops.SERVICES] + results.append(exercise(c, "rolling-update", ops.rolling_update)) + after = [ops.state(s) for s in ops.SERVICES] + check(all(a["container"] != b["container"] for a, b in zip(before, after)), + "both API containers must be replaced") + processes.append({"action": "rolling-update", "before": before, "after": after}) + (dest / "grade4.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (dest / "processes.json").write_text(json.dumps(processes, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + suffix = ".cmd" if os.name == "nt" else ".sh" + try: + checked("tests/resilience.py", "failover.txt", "--mode", "failover", + "--action-script", str(ROOT / ("scripts/stop-one" + suffix)), *http_args) + finally: + ops.restore() + checked("tests/resilience.py", "persistence.txt", "--mode", "persistence", + "--action-script", str(ROOT / ("scripts/restart" + suffix)), *http_args) + print("PASS: both replica failures, rolling replacement and confirmed data") + + +if __name__ == "__main__": + run(main) diff --git a/scripts/experiment_uploads.py b/scripts/experiment_uploads.py new file mode 100644 index 00000000..109f0942 --- /dev/null +++ b/scripts/experiment_uploads.py @@ -0,0 +1,71 @@ +"""10 simultaneous real uploads + Docker RSS observations (NO invented metrics).""" +import concurrent.futures +import json +import subprocess +import sys +import threading +import time +from pathlib import Path +from statistics import median +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, arguments, run + + +def mib(value): + value = value.strip() + for suffix, multiplier in (("GiB",1024),("MiB",1),("KiB",1/1024),("B",1/1048576)): + if value.endswith(suffix): + return float(value[:-len(suffix)]) * multiplier + return None + + +def main(): + args = arguments('Concurrent uploads + observed API container memory') + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = client.bootstrap() + file = (ROOT / 'tests/fixtures/sample.png').read_bytes() + results, memory = [], [] + stop = threading.Event() + def observe(): + while not stop.is_set(): + r = subprocess.run(['docker','stats','--no-stream','--format','{{.Name}}|{{.MemUsage}}'], + capture_output=True, text=True, timeout=10) + if r.returncode == 0: + for line in r.stdout.splitlines(): + if '|api1-' in line or '|api2-' in line: + pass + parts = line.split('|') + if len(parts)==2 and ('api1-' in parts[0] or 'api2-' in parts[0]): + amount = mib(parts[1].split('/')[0]) + if amount is not None: + memory.append({'container': parts[0], 'mem_mib': round(amount,3)}) + stop.wait(0.2) + monitor = threading.Thread(target=observe, daemon=True) + monitor.start() + try: + def worker(i): + local = Client(args.base_url,args.ca_file,timeout=30) + start=time.perf_counter() + resp=local.call('POST', f'/api/v1/chats/{chat["id"]}/attachments?filename=run{i}.png', + 201, actor=alice, raw=file, headers={'Content-Type':'image/png'}, schema='Attachment') + return {'elapsed_ms':round((time.perf_counter()-start)*1000,2), 'instance': resp['headers'].get('X-Instance-Id')} + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + futures=[executor.submit(worker,i) for i in range(10)] + results=[f.result() for f in concurrent.futures.as_completed(futures)] + finally: + stop.set(); monitor.join(timeout=12) + output={'concurrency':10,'file_bytes':len(file),'completed':len(results), + 'median_latency_ms':round(median(x['elapsed_ms'] for x in results),2), + 'api_peak_mem_mib_observed':max([x['mem_mib'] for x in memory],default=None), + 'memory_samples':len(memory),'instances':sorted(set(x['instance'] for x in results))} + target=ROOT / 'evidence' / 'uploads-10.json' + target.write_text(json.dumps(output,ensure_ascii=False,indent=2),encoding='utf-8') + check(len(results)==10, 'not all uploads completed') + print('PASS: 10 parallel uploads; metrics written:',target) + print(json.dumps(output, ensure_ascii=False, indent=2)) + if not memory: + print('WARNING: no Docker memory samples; run docker stats separately for a valid grade-4 experiment') + +if __name__ == '__main__': + run(main) diff --git a/scripts/ops.py b/scripts/ops.py new file mode 100644 index 00000000..344deb1e --- /dev/null +++ b/scripts/ops.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Operator actions for the local Compose stand; volumes are kept.""" +import argparse +import json +import os +from pathlib import Path +import subprocess +import time +import urllib.error +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +SERVICES = ("api1", "api2") +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + + +def compose(*args, capture=False): + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, + text=True, stdout=subprocess.PIPE if capture else None) + return result.stdout.strip() if capture else None + + +def state(service): + cid = compose("ps", "--all", "--quiet", service, capture=True) + if not cid: + raise RuntimeError("No container for " + service) + result = subprocess.check_output( + ["docker", "inspect", "--format", + '{{json .Config.Hostname}} {{json .State.Status}}', cid], text=True) + hostname, status = result.strip().split() + return {"service": service, "container": cid[:12], + "hostname": json.loads(hostname), "status": json.loads(status)} + + +def wait_instances(count=2, timeout=40): + deadline = time.monotonic() + timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=2) as r: + instance = r.headers.get("X-Instance-Id") + if instance: + seen.add(instance) + if len(seen) >= count: + return seen + except (OSError, urllib.error.URLError): + pass + time.sleep(0.2) + raise RuntimeError("Not enough ready API instances through LB") + + +def stop_one(service=None): + target = os.environ.get("TARGET_INSTANCE_ID", "").split(":", 1)[0] + if target: + service = next((s for s in SERVICES if state(s)["hostname"] == target), None) + if not service: + raise RuntimeError("TARGET_INSTANCE_ID is not an API container") + service = service or "api1" + if service not in SERVICES: + raise RuntimeError("Only API replicas may be stopped") + compose("kill", "--signal", "SIGKILL", service) + current = state(service) + if current["status"] != "exited": + raise RuntimeError("API did not stop") + print(json.dumps(current)) + + +def restore(): + compose("up", "-d", "--no-deps", "--no-build", "--wait", "--wait-timeout", "40", *SERVICES) + wait_instances() + + +def restart(): + compose("down") + compose("up", "-d", "--no-build", "--wait", "--wait-timeout", "60") + wait_instances() + + +def rolling_update(): + phases = [] + compose("build", "api1") + # This migration only adds tables and is compatible with the old API. + compose("run", "--rm", "--no-deps", "migrate") + for service in SERVICES: + before = state(service) + started = time.monotonic() + compose("up", "-d", "--no-deps", "--no-build", "--force-recreate", + "--wait", "--wait-timeout", "40", service) + after = state(service) + if before["container"] == after["container"]: + raise RuntimeError("Container was not replaced") + seen = wait_instances() + if not any(i.split(":", 1)[0] == after["hostname"] for i in seen): + raise RuntimeError("LB has not discovered the replacement") + phases.append({"service": service, "started": started, "finished": time.monotonic()}) + print(json.dumps({"replaced": service, "before": before, "after": after})) + return phases + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["restart", "stop-one", "restore", "rolling-update"]) + parser.add_argument("--service", choices=SERVICES) + args = parser.parse_args() + if args.action == "stop-one": + stop_one(args.service) + else: + {"restart": restart, "restore": restore, "rolling-update": rolling_update}[args.action]() + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..921e224b --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +python scripts\ops.py restart +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..da01f209 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/ops.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8c3cce50 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +python scripts\ops.py stop-one +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..c7b8ec50 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/ops.py stop-one diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 1dfe4a98..1542dc6a 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -6,17 +6,17 @@ from openapi_spec_validator import validate from jsonschema import Draft202012Validator root = Path(__file__).resolve().parents[1] -spec = json.loads((root/'contracts/openapi.json').read_text()) +spec = json.loads((root/'contracts/openapi.json').read_text(encoding='utf-8')) validate(spec) from referencing import Registry, Resource # The base URI resolves local #/components references inside examples. resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource)) -for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): +for name, example in json.loads((root/'contracts/examples.json').read_text(encoding='utf-8')).items(): schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name} Draft202012Validator(schema, registry=registry).validate(example) for path in (root/'contracts').glob('*.schema.json'): - schema = json.loads(path.read_text()) + schema = json.loads(path.read_text(encoding='utf-8')) Draft202012Validator.check_schema(schema) for example in schema.get('examples',[]): Draft202012Validator(schema).validate(example) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..d9e021d2 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,24 @@ +"""Entry point: uvicorn as PID 1 with env-driven host/port and graceful shutdown.""" +import os + +import uvicorn + +from .main import app + + +def main() -> None: + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run( + app, + host=host, + port=port, + log_level="info", + access_log=True, + timeout_keep_alive=2, + timeout_graceful_shutdown=5, + ) + + +if __name__ == "__main__": + main() diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..b9d77a42 --- /dev/null +++ b/src/auth.py @@ -0,0 +1,29 @@ +import os +import secrets +import redis.asyncio as redis +from passlib.context import CryptContext + +# bcrypt_sha256 avoids bcrypt's 72-byte limit. Old lab2 hashes still verify. +pwd_context = CryptContext(schemes=["bcrypt_sha256", "bcrypt"], deprecated="auto") +CACHE_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0") +SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +redis_client = redis.from_url(CACHE_URL, decode_responses=True, + socket_connect_timeout=1, socket_timeout=1) + +def hash_password(password: str) -> str: + return pwd_context.hash(password) + +def verify_password(plain_password: str, hashed_password: str) -> bool: + return pwd_context.verify(plain_password, hashed_password) + +async def create_session(user_id: str) -> str: + token = secrets.token_urlsafe(32) + await redis_client.setex(f"session:{token}", SESSION_TTL_SECONDS, user_id) + return token + +async def get_user_id_by_token(token: str) -> str | None: + return await redis_client.get(f"session:{token}") + +async def delete_session(token: str) -> None: + await redis_client.delete(f"session:{token}") diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..f7b984af --- /dev/null +++ b/src/db.py @@ -0,0 +1,91 @@ +import os +from datetime import datetime, timezone +from typing import List +from uuid import UUID, uuid4 + +from sqlalchemy import String, ForeignKey, DateTime, Index, Integer, BigInteger +from sqlalchemy.dialects.postgresql import UUID as PG_UUID, JSONB +from sqlalchemy.ext.asyncio import create_async_engine, async_sessionmaker +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql+asyncpg://user:password@localhost:5432/messenger") +engine = create_async_engine( + DATABASE_URL, echo=False, pool_pre_ping=True, pool_size=5, max_overflow=5, + pool_timeout=2, connect_args={"timeout": 1, "command_timeout": 3, + "server_settings": {"statement_timeout": "2500", "lock_timeout": "2000"}}, +) +async_session = async_sessionmaker(engine, expire_on_commit=False) + +class Base(DeclarativeBase): + pass + +class User(Base): + __tablename__ = "users" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + username: Mapped[str] = mapped_column(String(32), unique=True, index=True) + display_name: Mapped[str] = mapped_column(String(100)) + password_hash: Mapped[str] = mapped_column(String(255)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + +class Chat(Base): + __tablename__ = "chats" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + title: Mapped[str] = mapped_column(String(100)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + members: Mapped[List["ChatMember"]] = relationship(back_populates="chat", lazy="selectin") + +class ChatMember(Base): + __tablename__ = "chat_members" + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True) + user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True) + chat: Mapped["Chat"] = relationship(back_populates="members") + user: Mapped["User"] = relationship() + +class Message(Base): + __tablename__ = "messages" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True) + sender_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id")) + text: Mapped[str] = mapped_column(String(4000)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc)) + + # Составной индекс для быстрой курсорной пагинации (EXPLAIN покажет Index Scan) + __table_args__ = ( + Index('ix_messages_chat_created_id', 'chat_id', 'created_at', 'id'), + ) + +class IdempotencyKey(Base): + __tablename__ = "idempotency_keys" + user_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), primary_key=True) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), primary_key=True) + key: Mapped[str] = mapped_column(String(128), primary_key=True) + request_text: Mapped[str] = mapped_column(String(4000)) + response: Mapped[dict | None] = mapped_column(JSONB, nullable=True) + request_attachment_ids: Mapped[list] = mapped_column(JSONB, default=list, nullable=False) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + + +class SchemaVersion(Base): + __tablename__ = "schema_versions" + version: Mapped[int] = mapped_column(Integer, primary_key=True) + + +class Attachment(Base): + __tablename__ = "attachments" + id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), primary_key=True, default=uuid4) + chat_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("chats.id"), index=True, nullable=False) + owner_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + object_key: Mapped[str] = mapped_column(String(250), unique=True, nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(BigInteger, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column(String(16), default="ready", nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=lambda: datetime.now(timezone.utc), nullable=False) + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + message_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True) + attachment_id: Mapped[UUID] = mapped_column(PG_UUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True) + position: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..2fa4a33d --- /dev/null +++ b/src/images.py @@ -0,0 +1,47 @@ +"""Validate compressed image bytes without trusting filename or HTTP Content-Type.""" +import os +import threading +import struct +import zlib +from PIL import Image, ImageFile, UnidentifiedImageError +from .storage import MAX_IMAGE_PIXELS + +# Prevent unrestricted decompression of malicious image dimensions. +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS +ImageFile.LOAD_TRUNCATED_IMAGES = False +_IMAGE_LOCK = threading.BoundedSemaphore(1) + +class InvalidImage(Exception): + pass + +class TooManyPixels(Exception): + pass + + +def validate_image_file(file, declared_type: str) -> None: + expected = {"image/png": "PNG", "image/jpeg": "JPEG"}.get(declared_type) + if not expected: + raise InvalidImage("unsupported image type") + with _IMAGE_LOCK: + try: + file.seek(0) + with Image.open(file) as probe: + if probe.format != expected: + raise InvalidImage("image MIME does not match its content") + width, height = probe.size + if width <= 0 or height <= 0: + raise InvalidImage("image dimensions are invalid") + if width * height > MAX_IMAGE_PIXELS: + raise TooManyPixels("image dimensions exceed limit") + probe.verify() # verify compressed structure + file.seek(0) + with Image.open(file) as loaded: + loaded.load() # detect damaged truncated payloads + except Image.DecompressionBombError as e: + raise TooManyPixels("image dimensions exceed limit") from e + except Image.DecompressionBombWarning as e: + raise TooManyPixels("image dimensions exceed limit") from e + except (UnidentifiedImageError, OSError, ValueError, SyntaxError, EOFError, struct.error, zlib.error) as e: + raise InvalidImage("invalid or truncated PNG/JPEG") from e + finally: + file.seek(0) diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..abd8a342 --- /dev/null +++ b/src/main.py @@ -0,0 +1,540 @@ +from __future__ import annotations +import base64 +import hashlib +from tempfile import SpooledTemporaryFile +from botocore.exceptions import ClientError, BotoCoreError +import asyncio +import binascii +import logging +import os +import re +import socket +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone, timedelta +from typing import Optional + +from fastapi import Depends, FastAPI, Request, Query +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials +from pydantic import BaseModel, ConfigDict, Field, field_validator +from sqlalchemy import select, and_, or_, text +from sqlalchemy.dialects.postgresql import insert +from sqlalchemy.exc import IntegrityError, SQLAlchemyError +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from redis.exceptions import RedisError + +from asyncpg import PostgresError +from .db import engine, async_session, User, Chat, ChatMember, Message, IdempotencyKey, Attachment, MessageAttachment +from .auth import hash_password, verify_password, create_session, get_user_id_by_token, delete_session, redis_client, SESSION_TTL_SECONDS +from .storage import s3, bucket, check_storage, upload_original, delete_original, download_original, StorageFailure, MAX_UPLOAD_BYTES +from .images import validate_image_file, InvalidImage, TooManyPixels + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}", re.ASCII) +INSTANCE_ID = socket.gethostname() + ":" + str(os.getpid()) + +def now_iso(dt: datetime = None) -> str: + if not dt: + dt = datetime.now(timezone.utc) + return dt.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + +def error_payload(request_id: str, code: str, message: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status = status + self.code = code + self.message = message + super().__init__(message) + +# --- Курсоры для пагинации (с точностью до микросекунд) --- +def _encode_cursor(scope: str, created_at: datetime, obj_id: uuid.UUID) -> str: + raw = f"{scope}|{created_at.isoformat()}|{obj_id}".encode("utf-8") + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + +def _decode_cursor(cursor: str) -> tuple[str, datetime, uuid.UUID]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.urlsafe_b64decode(padded).decode("utf-8") + scope, dt_str, obj_id_str = raw.split("|", 2) + dt = datetime.fromisoformat(dt_str) + if dt.tzinfo is None: + raise ValueError("timezone required") + return scope, dt, uuid.UUID(obj_id_str) + except Exception: + raise ApiError(400, "invalid_request", "invalid cursor") + +# --- Схемы --- +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str = Field(strict=True) + display_name: str = Field(strict=True, min_length=1, max_length=100) + password: str = Field(strict=True, min_length=12, max_length=128) + + @field_validator("display_name", "password") + @classmethod + def not_blank(cls, value): + if not value.strip(): + raise ValueError("blank value") + return value + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str = Field(strict=True, min_length=1, max_length=100) + member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def not_blank(cls, value): + if not value.strip(): + raise ValueError("blank title") + return value + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str = Field(strict=True, max_length=4000) + attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def unique_attachments(cls, value): + if len(set(value)) != len(value): + raise ValueError("duplicate attachment id") + return value + + @field_validator("text") + @classmethod + def text_is_string(cls, value): + return value + + +@asynccontextmanager +async def lifespan(app: FastAPI): + yield + await redis_client.aclose() + await engine.dispose() + +app = FastAPI(title="Messenger", version="1.4.0", lifespan=lifespan) +security_bearer = HTTPBearer(auto_error=False) + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logging.getLogger(__name__).exception("Request failed: %s", request_id) + response = JSONResponse(status_code=500, content=error_payload(request_id, "internal_error", "internal error")) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + +def _rid(request: Request) -> str: + return getattr(request.state, "request_id", None) or str(uuid.uuid4()) + +# --- Обработчики ошибок --- +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + headers = {} + if exc.status == 401: + if request.url.path == "/api/v1/auth/sessions": + headers["WWW-Authenticate"] = 'Basic realm="messenger", charset="UTF-8"' + else: + headers["WWW-Authenticate"] = 'Bearer realm="messenger"' + return JSONResponse(status_code=exc.status, content=error_payload(_rid(request), exc.code, exc.message), headers=headers) + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return JSONResponse(status_code=400, content=error_payload(_rid(request), "invalid_request", "invalid request")) + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + return JSONResponse(status_code=exc.status_code, content=error_payload(_rid(request), "not_found", str(exc.detail))) + +@app.exception_handler(InvalidImage) +async def invalid_image_handler(request: Request, exc: InvalidImage): + return JSONResponse(status_code=415, content=error_payload(_rid(request), "unsupported_media_type", str(exc))) + +@app.exception_handler(TooManyPixels) +async def too_large_image_handler(request: Request, exc: TooManyPixels): + return JSONResponse(status_code=413, content=error_payload(_rid(request), "payload_too_large", str(exc))) + +@app.exception_handler(RedisError) +@app.exception_handler(SQLAlchemyError) +@app.exception_handler(PostgresError) +@app.exception_handler(OSError) +@app.exception_handler(TimeoutError) +@app.exception_handler(ClientError) +@app.exception_handler(BotoCoreError) +@app.exception_handler(StorageFailure) +async def dependency_error_handler(request: Request, exc: Exception): + # Если упала БД или Redis - отдаем 503, как требует контракт + return JSONResponse(status_code=503, content=error_payload(_rid(request), "unavailable", "dependency unavailable")) + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + return JSONResponse(status_code=500, content=error_payload(_rid(request), "internal_error", "internal error")) + +# --- Зависимости --- +async def get_actor(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)) -> dict: + if not credentials: + raise ApiError(401, "unauthorized", "missing token") + + user_id_str = await get_user_id_by_token(credentials.credentials) + if not user_id_str: + raise ApiError(401, "unauthorized", "invalid or expired token") + + async with async_session() as session: + user = await session.get(User, uuid.UUID(user_id_str)) + if not user: + raise ApiError(401, "unauthorized", "user not found") + return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + +# --- Роуты --- +@app.get("/health/live") +def liveness(): + return {"status": "ok"} + +@app.get("/health/ready") +async def readiness(): + async with asyncio.timeout(1.5): + async with async_session() as session: + version = await session.scalar(text("SELECT version FROM schema_versions WHERE version = 4")) + if version != 4: + raise ApiError(503, "unavailable", "migration required") + await redis_client.ping() + # Check S3 too: a ready API must be able to save and read originals. + await run_in_threadpool(check_storage) + return {"status": "ok"} + +@app.post("/api/v1/users", status_code=201) +async def register_user(payload: CreateUser): + if not USERNAME_RE.fullmatch(payload.username): + raise ApiError(400, "invalid_request", "invalid username") + + async with async_session() as session: + existing = await session.scalar(select(User).where(User.username == payload.username)) + if existing: + raise ApiError(409, "conflict", "username already exists") + + password_hash = await run_in_threadpool(hash_password, payload.password) + user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash) + session.add(user) + try: + await session.commit() + except IntegrityError: + await session.rollback() + raise ApiError(409, "conflict", "username already exists") + + return {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session_endpoint(request: Request): + # HTTP Basic carries a UTF-8 password, not only ASCII. + try: + scheme, encoded = request.headers.get("Authorization", "").split(" ", 1) + if scheme.lower() != "basic": + raise ValueError("Basic required") + username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + raise ApiError(401, "unauthorized", "invalid credentials") + + async with async_session() as session: + user = await session.scalar(select(User).where(User.username == username)) + if not user or not await run_in_threadpool(verify_password, password, user.password_hash): + raise ApiError(401, "unauthorized", "invalid credentials") + + token = await create_session(str(user.id)) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL_SECONDS) + return { + "token": token, + "token_type": "Bearer", + "expires_at": now_iso(expires_at), + "user": {"id": str(user.id), "username": user.username, "display_name": user.display_name, "created_at": now_iso(user.created_at)} + } + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(credentials: HTTPAuthorizationCredentials = Depends(security_bearer)): + if not credentials: + raise ApiError(401, "unauthorized", "missing token") + + user_id = await get_user_id_by_token(credentials.credentials) + if not user_id: + raise ApiError(401, "unauthorized", "invalid or expired token") + + await delete_session(credentials.credentials) + +@app.get("/api/v1/users/me") +async def current_user(actor: dict = Depends(get_actor)): + return actor + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(payload: CreateChat, actor: dict = Depends(get_actor)): + member_ids = [str(u) for u in payload.member_ids] + if actor["id"] not in member_ids: + raise ApiError(400, "invalid_request", "creator must be a member") + if len(set(member_ids)) != len(member_ids): + raise ApiError(400, "invalid_request", "duplicate members") + + async with async_session() as session: + users = await session.scalars(select(User.id).where(User.id.in_([uuid.UUID(m) for m in member_ids]))) + if len(list(users)) != len(set(member_ids)): + raise ApiError(400, "invalid_request", "unknown member") + + chat = Chat(title=payload.title) + session.add(chat) + await session.flush() + + for mid in member_ids: + session.add(ChatMember(chat_id=chat.id, user_id=uuid.UUID(mid))) + + await session.commit() + return {"id": str(chat.id), "title": chat.title, "member_ids": member_ids, "created_at": now_iso(chat.created_at)} + +@app.get("/api/v1/chats") +async def list_chats(limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)): + async with async_session() as session: + stmt = select(Chat).join(ChatMember).where(ChatMember.user_id == uuid.UUID(actor["id"])) + if cursor: + scope, c_dt, c_id = _decode_cursor(cursor) + if scope != "chats": + raise ApiError(400, "invalid_request", "invalid cursor scope") + stmt = stmt.where(or_( + Chat.created_at > c_dt, + and_(Chat.created_at == c_dt, Chat.id > c_id) + )) + + stmt = stmt.order_by(Chat.created_at.asc(), Chat.id.asc()).limit(limit + 1) + chats = (await session.scalars(stmt)).unique().all() + + has_more = len(chats) > limit + page = chats[:limit] + next_cursor = _encode_cursor("chats", page[-1].created_at, page[-1].id) if has_more and page else None + + items = [] + for c in page: + items.append({ + "id": str(c.id), + "title": c.title, + "created_at": now_iso(c.created_at), + "member_ids": [str(m.user_id) for m in c.members] # Теперь участники возвращаются корректно + }) + + return {"items": items, "next_cursor": next_cursor} + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: uuid.UUID, actor: dict = Depends(get_actor)): + async with async_session() as session: + chat = await session.scalar(select(Chat).join(ChatMember).where(Chat.id == chat_id, ChatMember.user_id == uuid.UUID(actor["id"]))) + if not chat: + raise ApiError(404, "not_found", "chat not found") + return {"id": str(chat.id), "title": chat.title, "member_ids": [str(m.user_id) for m in chat.members], "created_at": now_iso(chat.created_at)} + +async def require_chat_member(session, chat_id: uuid.UUID, user_id: str): + member = await session.scalar(select(ChatMember).where(and_( + ChatMember.chat_id == chat_id, ChatMember.user_id == uuid.UUID(user_id)))) + if not member: + raise ApiError(404, "not_found", "chat not found") + + +def attachment_json(item: Attachment) -> dict: + return { + "id": str(item.id), "chat_id": str(item.chat_id), + "owner_id": str(item.owner_id), "filename": item.filename, + "content_type": item.content_type, "size_bytes": item.size_bytes, + "sha256": item.sha256, "status": item.status, + "variants": [], "error_code": None, + "created_at": now_iso(item.created_at), "updated_at": now_iso(item.updated_at), + } + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: uuid.UUID, request: Request, filename: str = Query(...), + actor: dict = Depends(get_actor), +): + # Authorize before revealing whether the resource exists. + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + + if not (1 <= len(filename) <= 255) or any( + c in filename for c in ("/", "\\") + ) or any(ord(c) < 32 or ord(c) == 127 for c in filename): + raise ApiError(400, "invalid_request", "invalid filename") + + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "only PNG and JPEG are accepted") + + # Stream request in bounded chunks; spooling moves larger input to /tmp. + hasher = hashlib.sha256() + size = 0 + with SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as original: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise ApiError(413, "payload_too_large", "image exceeds 10 MiB") + hasher.update(chunk) + original.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "empty image") + await run_in_threadpool(validate_image_file, original, declared_type) + + item_id = uuid.uuid4() + # Random server-generated object key, never derived from filename. + key = f"originals/{item_id.hex}/{uuid.uuid4().hex}" + await run_in_threadpool(upload_original, original, key, size, declared_type) + try: + async with async_session() as session: + # Recheck membership before COMMIT in case policy changes meanwhile. + await require_chat_member(session, chat_id, actor["id"]) + item = Attachment( + id=item_id, chat_id=chat_id, owner_id=uuid.UUID(actor["id"]), + filename=filename, object_key=key, content_type=declared_type, + size_bytes=size, sha256=hasher.hexdigest(), status="ready", + ) + session.add(item) + await session.commit() + result = attachment_json(item) + except Exception: + # Best-effort compensation. Crashes before this point are recovered + # by `python -m src.reconcile --apply` after the safety grace period. + try: + await run_in_threadpool(delete_original, key) + except Exception: + logging.getLogger(__name__).warning("S3 compensation failed; reconcile needed") + raise + return result + + +@app.get("/api/v1/attachments/{attachment_id}") +async def attachment_metadata(attachment_id: uuid.UUID, actor: dict = Depends(get_actor)): + async with async_session() as session: + item = await session.get(Attachment, attachment_id) + if item is None: + raise ApiError(404, "not_found", "attachment not found") + await require_chat_member(session, item.chat_id, actor["id"]) + return attachment_json(item) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def attachment_bytes( + attachment_id: uuid.UUID, variant: str = "original", actor: dict = Depends(get_actor) +): + async with async_session() as session: + item = await session.get(Attachment, attachment_id) + if item is None: + raise ApiError(404, "not_found", "attachment not found") + await require_chat_member(session, item.chat_id, actor["id"]) + if variant != "original": + raise ApiError(404, "not_found", "variant not found") + if item.status != "ready": + raise ApiError(409, "conflict", "attachment is not ready") + key, content_type, size = item.object_key, item.content_type, item.size_bytes + + body = await run_in_threadpool(download_original, key) + + def chunks(): + try: + yield from body.iter_chunks(chunk_size=64 * 1024) + finally: + body.close() + + return StreamingResponse(chunks(), media_type=content_type, + headers={"Content-Length": str(size), + "Cache-Control": "private, no-store"}) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def post_message(chat_id: uuid.UUID, payload: CreateMessage, request: Request, actor: dict = Depends(get_actor)): + key = request.headers.get("Idempotency-Key", "") + if not KEY_RE.fullmatch(key): + raise ApiError(400, "invalid_request", "invalid Idempotency-Key") + if not payload.text.strip() and not payload.attachment_ids: + raise ApiError(400, "invalid_request", "empty message") + attachment_ids = [str(a) for a in payload.attachment_ids] + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + + # Both original and repeated messages must satisfy access constraints. + if attachment_ids: + attachments = (await session.scalars(select(Attachment).where( + Attachment.id.in_(payload.attachment_ids), Attachment.chat_id == chat_id + ))).all() + if len(attachments) != len(attachment_ids): + raise ApiError(404, "not_found", "attachment not found") + if any(a.status != "ready" for a in attachments): + raise ApiError(409, "conflict", "attachment not ready") + + now = datetime.now(timezone.utc) + scope = {"user_id": uuid.UUID(actor["id"]), "chat_id": chat_id, "key": key} + await session.execute(insert(IdempotencyKey).values( + **scope, request_text=payload.text, request_attachment_ids=attachment_ids, + expires_at=now + timedelta(hours=24) + ).on_conflict_do_nothing()) + record = await session.scalar(select(IdempotencyKey).where( + IdempotencyKey.user_id == scope["user_id"], + IdempotencyKey.chat_id == chat_id, IdempotencyKey.key == key + ).with_for_update()) + if record.response is not None and record.expires_at > now: + if record.request_text != payload.text or record.request_attachment_ids != attachment_ids: + raise ApiError(409, "conflict", "key used with another message") + return record.response + + msg = Message(chat_id=chat_id, sender_id=uuid.UUID(actor["id"]), text=payload.text) + session.add(msg) + await session.flush() + for pos, a_id in enumerate(payload.attachment_ids): + session.add(MessageAttachment(message_id=msg.id, attachment_id=a_id, position=pos)) + result = { + "id": str(msg.id), "chat_id": str(msg.chat_id), "sender_id": str(msg.sender_id), + "text": msg.text, "created_at": now_iso(msg.created_at), + "attachment_ids": attachment_ids, + } + record.request_text = payload.text + record.request_attachment_ids = attachment_ids + record.expires_at = now + timedelta(hours=24) + record.response = result + await session.commit() + return result + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def get_messages(chat_id: uuid.UUID, limit: int = Query(50, ge=1, le=100), cursor: Optional[str] = None, actor: dict = Depends(get_actor)): + async with async_session() as session: + await require_chat_member(session, chat_id, actor["id"]) + stmt = select(Message).where(Message.chat_id == chat_id) + if cursor: + scope, c_dt, c_id = _decode_cursor(cursor) + if scope != str(chat_id): + raise ApiError(400, "invalid_request", "invalid cursor scope") + stmt = stmt.where(or_( + Message.created_at > c_dt, + and_(Message.created_at == c_dt, Message.id > c_id) + )) + stmt = stmt.order_by(Message.created_at.asc(), Message.id.asc()).limit(limit + 1) + messages = (await session.scalars(stmt)).all() + has_more = len(messages) > limit + page = messages[:limit] + next_cursor = _encode_cursor(str(chat_id), page[-1].created_at, page[-1].id) if has_more and page else None + + msg_ids = [m.id for m in page] + links = [] + if msg_ids: + links = (await session.execute(select(MessageAttachment).where( + MessageAttachment.message_id.in_(msg_ids) + ).order_by(MessageAttachment.message_id, MessageAttachment.position))).scalars().all() + linked: dict[uuid.UUID, list[str]] = {m.id: [] for m in page} + for link in links: + linked[link.message_id].append(str(link.attachment_id)) + items = [{ + "id": str(m.id), "chat_id": str(m.chat_id), "sender_id": str(m.sender_id), + "text": m.text, "created_at": now_iso(m.created_at), + "attachment_ids": linked[m.id], + } for m in page] + return {"items": items, "next_cursor": next_cursor} diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..37f55cf5 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,21 @@ +import asyncio + +from sqlalchemy import text +from .db import Base, engine + + +async def main(): + async with engine.begin() as conn: + # Only the migrate service creates tables; concurrent operators serialize. + await conn.execute(text("SELECT pg_advisory_xact_lock(300300)")) + # Schema upgrade preserves all existing lab3 data. + await conn.run_sync(Base.metadata.create_all) + await conn.execute(text("ALTER TABLE idempotency_keys ADD COLUMN IF NOT EXISTS request_attachment_ids JSONB NOT NULL DEFAULT '[]'::jsonb")) + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (3) ON CONFLICT DO NOTHING")) + await conn.execute(text("INSERT INTO schema_versions(version) VALUES (4) ON CONFLICT DO NOTHING")) + await engine.dispose() + print("Migration 4 applied: existing lab3 tables kept, attachments added") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/private_check.py b/src/private_check.py new file mode 100644 index 00000000..72d04d6d --- /dev/null +++ b/src/private_check.py @@ -0,0 +1,22 @@ +"""Prove S3 is not public and API-scoped credentials cannot list other buckets.""" +import urllib.error +import urllib.request +from botocore.exceptions import ClientError +from .storage import s3, bucket +import os +import uuid + +url = os.getenv("S3_ENDPOINT", "http://s3:8333").rstrip("/") + "/" + bucket + "/random-inaccessible-key" +try: + urllib.request.urlopen(url, timeout=4) +except urllib.error.HTTPError as exc: + assert exc.code in (401, 403), f"expected S3 denied, got {exc.code}" +else: + raise AssertionError("UNSAFE: direct unsigned S3 request succeeded") +try: + s3.create_bucket(Bucket="forbidden-probe-" + uuid.uuid4().hex[:16]) +except ClientError as exc: + assert exc.response["Error"]["Code"] in ("AccessDenied", "403"), "unexpected S3 result" +else: + raise AssertionError("UNSAFE: API credentials have global create-bucket privileges") +print("PASS: S3 requires authentication; API credentials restricted to own bucket") diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..a5f9e9d3 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,51 @@ +"""Safe orphan S3 cleanup (dry run default). Use inside api container. + +A crash between S3 PUT and PostgreSQL COMMIT can leave an unreferenced object. +Only objects older than the grace period are candidates. No deletion if DB read fails. +""" +import argparse +import asyncio +from datetime import datetime, timezone, timedelta +from sqlalchemy import select +from .db import async_session, Attachment, engine +from .storage import s3, bucket +from starlette.concurrency import run_in_threadpool + + +def list_s3(): + keys = [] + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=bucket, Prefix="originals/"): + keys.extend((i["Key"], i["LastModified"]) for i in page.get("Contents", [])) + return keys + + +async def run(apply=False, grace_seconds=3600): + if grace_seconds < 3600: + raise ValueError("grace must be at least 3600 seconds") + # Load DB references before enumerating/deleting S3 objects. Fail closed. + async with async_session() as session: + referenced = set((await session.scalars(select(Attachment.object_key))).all()) + objects = await run_in_threadpool(list_s3) + threshold = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + candidates = [key for key, when in objects if key not in referenced and when < threshold] + print(f"referenced={len(referenced)} objects={len(objects)} old_orphans={len(candidates)} apply={apply}") + for key in candidates: + # Avoid listing filenames/secrets: keys are random IDs, not user inputs. + if apply: + await run_in_threadpool(s3.delete_object, Bucket=bucket, Key=key) + print(("deleted " if apply else "would_delete ") + key) + return len(candidates) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="really delete aged orphan objects") + parser.add_argument("--grace-seconds", type=int, default=3600) + args = parser.parse_args() + async def main(): + try: + await run(args.apply, args.grace_seconds) + finally: + await engine.dispose() + asyncio.run(main()) diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..45ba7d89 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,43 @@ +"""S3-compatible private storage. Only API-scoped credentials are available here.""" +import os +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + +MAX_UPLOAD_BYTES = int(os.getenv("MAX_UPLOAD_BYTES", "10485760")) +MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000")) +bucket = os.getenv("S3_BUCKET", "messenger-attachments") +s3 = boto3.client( + "s3", endpoint_url=os.getenv("S3_ENDPOINT", "http://s3:8333"), + region_name=os.getenv("S3_REGION", "us-east-1"), + aws_access_key_id=os.environ.get("S3_ACCESS_KEY_ID", "unset"), + aws_secret_access_key=os.environ.get("S3_SECRET_ACCESS_KEY", "unset"), + config=Config(s3={"addressing_style": "path"}, connect_timeout=2, read_timeout=30, + retries={"mode": "standard", "max_attempts": 2}), +) + +class StorageFailure(Exception): + pass + + +def check_storage(): + # Scoped user has ListBucket but no privileges outside this bucket. + s3.list_objects_v2(Bucket=bucket, MaxKeys=1) + + +def upload_original(file, key, size, content_type): + file.seek(0) + s3.put_object(Bucket=bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + + +def delete_original(key): + s3.delete_object(Bucket=bucket, Key=key) + + +def download_original(key): + try: + return s3.get_object(Bucket=bucket, Key=key)["Body"] + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404"): + raise StorageFailure("Object missing") from None + raise diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..588ebe22 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import sys +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Lab3: concurrent idempotency and validation") + c = Client(args.base_url, args.ca_file) + c.call("POST", "/api/v1/users", 400, + body={"username": "valid\n", "display_name": "Test", "password": "LongPassword123"}) + registration = {"username": "race_" + uuid.uuid4().hex[:12], + "display_name": "Race", "password": "LongPassword123"} + registration_barrier = threading.Barrier(2) + + def register(_): + local = Client(args.base_url, args.ca_file) + registration_barrier.wait() + return local.call("POST", "/api/v1/users", [201, 409], + body=registration, schema="User")["status"] + + with ThreadPoolExecutor(max_workers=2) as pool: + check(sorted(pool.map(register, range(2))) == [201, 409], "registration race") + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/" + chat["id"] + "/messages" + key = uuid.uuid4().hex + barrier = threading.Barrier(12) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.message(alice, chat, "Одна запись", key=key) + + with ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(send, range(12))) + first = responses[0]["json"] + check(all(r["json"] == first for r in responses), "concurrent replay differs") + instances = {r["headers"]["X-Instance-Id"] for r in responses} + check(len(instances) >= 2, "same key must reach at least two replicas") + page = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(page["items"] == [first], "concurrent retry created duplicates") + c.call("POST", path, 409, actor=alice, body={"text": "Изменённый текст"}, + headers={"Idempotency-Key": key}) + c.call("POST", path, 404, actor=eve, body={"text": "Одна запись"}, + headers={"Idempotency-Key": key}) + # The same key belongs to a different scope for Bob and for another chat. + other = c.message(bob, chat, "Другая область", key=key)["json"] + check(other["id"] != first["id"], "different users share an idempotency scope") + second_chat = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Второй", "member_ids": [alice["id"]]}, + schema="Chat")["json"] + other = c.message(alice, second_chat, "Одна запись", key=key)["json"] + check(other["id"] != first["id"], "different chats share an idempotency scope") + for bad_key in ("", "bad key", "x" * 129): + c.call("POST", path, 400, actor=alice, body={"text": "Текст"}, + headers={"Idempotency-Key": bad_key}) + for body in ({"text": " "}, {"text": 123}, {"text": "x", "sender_id": alice["id"]}): + c.call("POST", path, 400, actor=alice, body=body, + headers={"Idempotency-Key": uuid.uuid4().hex}) + c.call("POST", path, 400, actor=alice, raw=b'{"text":', + headers={"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex}) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + c.call("POST", path, 401, actor=alice, body={"text": "Одна запись"}, + headers={"Idempotency-Key": key}) + print("PASS: 12 concurrent requests across %d replicas, one message" % len(instances)) + print("PASS: conflict, key scope, validation, access and revoked session") + print("PASS: concurrent registration, exactly one 201 and one 409") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..9f294fd1 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Additional grade-4 negative and access tests. Requires running Compose stack.""" +import hashlib +import zlib +import sys +import uuid +from pathlib import Path +from concurrent.futures import ThreadPoolExecutor +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, arguments, run + +ROOT = Path(__file__).resolve().parents[1] + +def main(): + args = arguments("Lab4: file validation, permissions, idempotency and isolation") + c = Client(args.base_url, args.ca_file, timeout=35) + alice, bob, eve, chat = c.bootstrap() + file = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + upload = path + "/attachments?filename=photo.png" + result = c.call("POST",upload,201,actor=alice,raw=file, + headers={"Content-Type":"image/png"},schema="Attachment")["json"] + obj = "/api/v1/attachments/" + result["id"] + check(result["sha256"] == hashlib.sha256(file).hexdigest(),"checksum mismatch") + check(c.call("GET",obj+"/content",actor=bob)["bytes"] == file,"original differs") + c.call("GET",obj,404,actor=eve) + c.call("GET",obj+"/content",404,actor=eve) + c.call("GET","/api/v1/attachments/"+str(uuid.uuid4()),404,actor=bob) + c.call("POST",upload,415,actor=alice,raw=b"not an image", + headers={"Content-Type":"image/png"}) + c.call("POST",upload,415,actor=alice,raw=file, + headers={"Content-Type":"image/jpeg"}) + c.call("POST",upload,413,actor=alice,raw=b"\0"*10485761, + headers={"Content-Type":"image/png"}) + huge = bytearray(file) + huge[16:20] = (10000).to_bytes(4, "big") + huge[20:24] = (5000).to_bytes(4, "big") + huge[29:33] = (zlib.crc32(huge[12:29]) & 0xffffffff).to_bytes(4, "big") + c.call("POST",upload,413,actor=alice,raw=bytes(huge), + headers={"Content-Type":"image/png"}) + c.call("POST",path+"/attachments?filename=..%2Fhidden.png",400,actor=alice, + raw=file,headers={"Content-Type":"image/png"}) + c.call("POST",path+"/attachments?filename=sample.txt",415,actor=alice, + raw=file,headers={"Content-Type":"text/plain"}) + second = c.call("POST","/api/v1/chats",201,actor=alice, + body={"title":"Other Chat","member_ids":[alice["id"]]},schema="Chat")["json"] + c.call("POST", "/api/v1/chats/"+second["id"]+"/messages",404, + actor=alice,body={"text":"","attachment_ids":[result["id"]]}, + headers={"Idempotency-Key":uuid.uuid4().hex}) + attached = c.message(alice,chat,"",attachments=[result["id"]])["json"] + check(attached["attachment_ids"] == [result["id"]],"message lost attachment") + key = uuid.uuid4().hex + one = c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"] + check(c.message(alice,chat,"Фото",key=key,attachments=[result["id"]])["json"] == one, + "repeated idempotency response mismatch") + c.call("POST",path+"/messages",409,actor=alice, + body={"text":"Фото", "attachment_ids":[]},headers={"Idempotency-Key":key}) + print("PASS: private downloads, MIME spoof, corrupt image, oversize, dangerous filename, foreign chat, idempotency") + +if __name__ == "__main__": + run(main) diff --git a/tests/client.py b/tests/client.py index f0499a29..977b2842 100644 --- a/tests/client.py +++ b/tests/client.py @@ -13,8 +13,8 @@ import urllib.request import uuid ROOT = Path(__file__).resolve().parents[1] -LAB = json.loads((ROOT / 'lab.json').read_text())['number'] -SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text()) +LAB = json.loads((ROOT / 'lab.json').read_text(encoding='utf-8'))['number'] +SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text(encoding='utf-8')) class Failure(AssertionError): pass @@ -102,7 +102,7 @@ class Client: self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/') check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)') check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS') - self.timeout = timeout + self.timeout = max(timeout, 5) context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None) self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))