ЛР 4 — БПИ2301 — Дугина — на 4 #82

Closed
aleksandra_dugina wants to merge 1 commits from aleksandra_dugina/mtusi2026lab4:solution/BPI2301-Dugina into main
67 changed files with 3436 additions and 46 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
DEFENSE.md
student-tests
+12 -19
View File
@@ -1,20 +1,13 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# Local settings. Generate .env and secrets with: python3 scripts/init-local.py
COMPOSE_PROJECT_NAME=mtusi2026lab4
PUBLIC_PORT=8084
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
# Compose supplies these paths/settings to containers; secret values never go here.
# DATABASE_URL_FILE=/run/secrets/database_url
# CACHE_URL=redis://redis:6379/0 (profile cache; sessions are authoritative in PostgreSQL)
# S3_ENDPOINT=http://s3:9000
# S3_REGION=us-east-1
# S3_BUCKET=messenger-attachments
# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
# Fixed contract limits: 10485760 bytes, 40000000 pixels.
+2
View File
@@ -30,3 +30,5 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
DEFENSE.md
+30
View File
@@ -0,0 +1,30 @@
FROM python:3.11.16-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV APP_HOST=0.0.0.0
ENV APP_PORT=8080
WORKDIR /app
COPY requirements.txt .
RUN python -m pip install --no-cache-dir -r requirements.txt
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
COPY --chown=appuser:appuser src/ ./src/
COPY --chown=appuser:appuser db/ ./db/
COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
USER appuser
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2", "--limit-concurrency", "64"]
+267 -23
View File
@@ -1,37 +1,281 @@
# Отчёт — лабораторная 4
# Лабораторная №4 — S3-хранилище и вложения
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Автор: **Дугина Александра, БПИ2301**. Дисциплина: «Высоконагруженные приложения», МТУСИ.
Целевая оценка: **4**.
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker Engine/Compose v2, Python 3.9+ для публичных проверок и make.
В контейнере используется Python 3.11.16. На macOS должен работать Docker Desktop.
## Архитектура и решения
```sh
python3 scripts/init-local.py
python3 -m venv .venv
.venv/bin/python -m pip install -r requirements-dev.txt
make check
make validate PYTHON=.venv/bin/python
# На macOS для credential helper при необходимости:
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build
scripts/compose.sh ps
curl -i http://localhost:8084/health/ready
make test BASE_URL=http://localhost:8084
make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
scripts/compose.sh start api
```
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
`init-local.py` создаёт локальные случайные пароли, файлы secrets с mode 0600
и `.env`, не перезаписывая имеющуюся конфигурацию. `.env` и secrets исключены
из Git и контекста Docker build. Пароли и токены в evidence не сохраняются.
Оба bootstrap запускаются Compose автоматически; повторный `up` безопасен.
Swagger с обязательным контрактом: `http://localhost:8084/docs`.
## Доказательства критериев
Полный проверочный прогон: `python3 scripts/verify.py`. Он намеренно выполняет
отказы только lab4, создаёт синтетические данные и восстанавливает сервисы.
Отдельные команды описаны в [scripts/README.md](scripts/README.md).
Безопасная остановка: `scripts/compose.sh stop`; возобновление:
`scripts/compose.sh up -d`. Скрипты не удаляют volumes или пользовательские данные.
Команд удаления volumes в инструкции и служебных скриптах нет.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
## Переход от lab3
Источник — собственный соседний `../mtusi2026lab3`, только чтение. Перенесены:
`src/auth.py`, `database.py`, `user_repository.py`, `session_cache.py`,
основа `main.py` и `chat_repository.py`, SQL-миграции 001/002, Dockerfile,
requirements и HAProxy/Compose. Сохранены Argon2id, Basic-вход, непрозрачные
256-битные Bearer-токены, PostgreSQL-сессии с TTL, Redis-кеш профилей,
чаты/участники, курсорная история, транзакционная идемпотентность сообщений,
балансировка, обязательные заголовки и healthchecks.
В lab4 адаптированы сообщения к `attachment_ids`, добавлены S3 и очистка.
Старые POST `{"text":"..."}` эквивалентны запросам с `attachment_ids: []`.
При конкурентном повторе ключа через разные реплики создаётся одно сообщение;
сравнение повторного body включает текст и упорядоченный список вложений.
Сообщение и запись идемпотентности фиксируются одной транзакцией.
Срок хранения ключей не ограничен уборщиком: гарантируется минимум требуемых 24 часов.
Исходный lab3, его `.env`, secrets, volumes, venv и evidence не изменялись и не
копировались как результаты lab4. Публичные `tests/`, `contracts/`, требования
и критерии текущего skeleton не изменены. Все новые тесты — в `student-tests/`.
## Архитектура и версии
```text
Клиент → 127.0.0.1:8084 → HAProxy → FastAPI × 2
├─ PostgreSQL: данные, сессии, метаданные
├─ Redis: необязательный кеш профилей
└─ MinIO: оригинальные PNG/JPEG
cleanup → PostgreSQL + MinIO
```
Выбран MinIO: стандартный S3 API, SDK boto3, приватные bucket и policy,
раздельные credentials, простой persistent volume. Используется проверенный
в этом стенде **Bitnami 2025.7.23-debian-12-r5, MinIO DEVELOPMENT.2025-07-23T15-54-02Z**, закреплённый по digest.
Первоначальные попытки скачать release-теги и образ Quay по digest завершились
отказами реестров. Quay-образ работал из локального кеша, но не подходил для
чистого запуска; поэтому итоговый S3 и bootstrap используют один доступный
Bitnami-образ. Его pull по точному digest реально выполнен повторно и включён
в итоговый verify. Перед сменой образа сохранена локальная резервная копия
volume в игнорируемом backups/, после смены проверены все 33 прежних оригинала
(79 300 697 bytes), все SHA-256 совпали — storage-image-switch.txt.
Смена образа не создавала новый пустой volume и не удаляла старые данные.
| Роль / service | Образ / runtime | Health / запуск | Persistent state | Лимит CPU / RAM |
| --- | --- | --- | --- | --- |
| Балансировщик `lb` | haproxy:3.0.9-alpine | readiness API | нет | 0,5 / 128 MiB |
| `api` × 2 | python:3.11.16-slim-bookworm, собственная сборка | PostgreSQL + LIST S3 | нет; tmpfs 16 MiB | каждый 0,75 / 768 MiB |
| `postgres` | postgres:16.8-bookworm | pg_isready | postgres_data | 1 / 512 MiB |
| `redis` | redis:7.4.2-alpine | ping | redis_data, AOF always | 0,25 / 128 MiB |
| `s3` | bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 | /minio/health/live | s3_data | 1 / 768 MiB |
| `s3-bootstrap` | bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 | одноразовый mc | нет | одноразовая задача |
| `migrate` | образ API | SQL под advisory lock | schema_migrations в БД | 0,5 / 256 MiB |
| `cleanup` | образ API | цикл 300 с, restart unless-stopped | нет | 0,25 / 256 MiB |
Compose project — `mtusi2026lab4`; volumes получают этот prefix. Lab3 остаётся
на 8080. Только LB публикуется и только на loopback; порты реплик, PostgreSQL,
Redis, S3 9000 и консоли 9001 наружу не публикуются. Сеть — отдельная
`mtusi2026lab4_default`.
Библиотеки: FastAPI 0.115.12, Uvicorn 0.34.2, psycopg 3.2.6,
psycopg-pool 3.2.6, Redis 5.2.1, argon2-cffi 23.1.0,
boto3 1.38.12, Pillow 11.2.1; фактически установлен botocore 1.38.46.
`.env.example` содержит настройки запуска; внутренние URL и пути secrets заданы
в Compose. Контрактные лимиты фиксированы в коде, не повышаются через env.
Redis не источник сессий: каждый защищённый запрос проверяет сессию в PostgreSQL.
Отказ кеша не делает API неготовым; PostgreSQL и S3 критичны для readiness.
## Данные и API
Миграция `003_attachments.sql` добавляет таблицу Attachment: UUID, chat_id,
owner_id, filename, уникальный object_key, size_bytes, реальный content_type,
SHA-256, ready status, created_at и updated_at. Внешние ключи связывают её с
пользователем и чатом. Сообщение хранит упорядоченный массив UUID вложений;
при записи в той же транзакции проверяется существование всех ID в этом чате.
Обязательный API не удаляет вложения и не меняет участников чата.
Бинарные данные хранятся только в S3 `messenger-attachments/originals/<server UUID>`.
Имя пользователя не является путём или ключом. API не возвращает object_key,
S3 endpoint или credentials. JSON вложения содержит `variants: []`,
`error_code: null`, `status: ready` и все поля обязательного OpenAPI.
Загрузка: `POST /api/v1/chats/{chat_id}/attachments?filename=photo.png`,
raw binary, не multipart. Обязательны Bearer и участие в чате.
Размер проверяется по Content-Length до чтения и по каждому chunk при чтении;
chunked upload также ограничен 10 MiB. Байты поступают во временный файл tmpfs,
одновременно вычисляется SHA-256. На реплике только одна активная загрузка;
остальные ждут semaphore максимум 30 с без полного чтения тела.
Декодер работает в отдельном Linux-процессе: Pillow проверяет формат, размеры,
PNG CRC/структуру и полное декодирование. До load проверяется 40 млн пикселей.
У процесса RLIMIT_AS 512 MiB, CPU 8 с, wall timeout 10 с; анимированные изображения
не принимаются. В API ограничены tmpfs и память контейнера; Uvicorn допускает
не более 64 конкурентных соединений. На чтение upload отведено 30 с.
Ни файл больше 10 MiB, ни декомпрессионная бомба не читаются/декодируются без лимита.
Временный файл закрывается и удаляется; после SIGKILL tmpfs не является persistent
хранилищем и очищается при остановке контейнера.
Filename длиной 1–255 символов не может быть пустым/пробельным, содержать
slash/backslash, NUL или управляющие/служебные символы. Несоответствующий MIME,
битое изображение — 415, превышение bytes/pixels — 413, неверные поля — 400.
JSON-запросы тоже читаются с лимитом 64 KiB.
Скачивание идёт через `GET /api/v1/attachments/{id}/content` без redirect,
потоками по 64 KiB; SDK использует connect timeout 2 с, read timeout 5 с,
не повторяет PUT автоматически. Ответ содержит истинный MIME, Content-Length,
Content-Disposition: attachment, nosniff и private/no-store. Оригинал не
перекодируется: побайтовое равенство и SHA-256 проверены для PNG и JPEG.
Участник читает metadata/content; посторонний/неизвестный ID получает 404,
неавторизованный — 401 с WWW-Authenticate. Ошибки API сохраняют X-Request-Id,
X-Instance-Id и контрактный JSON без секретов/SQL/stack trace.
Сообщение принимает максимум 10 уникальных ID. Пустой текст допустим только
с вложением. Неизвестные, чужие и принадлежащие другому чату ID — 404.
Любой участник может использовать вложение своего чата, включая загруженное
другим участником. Обязательная курсорная пагинация `(created_at,id)` сохранена;
это не snapshot истории между запросами, при конкурентных вставках действует
обычная keyset-семантика и серверное время PostgreSQL.
## Транзакционные границы и восстановление
Общей атомарной транзакции S3/PostgreSQL нет. Порядок:
1. Полностью принять и проверить файл; при обрыве до этого шага PUT не выполняется.
2. Открыть транзакцию БД и взять advisory lock по серверному object_key.
3. Выполнить один S3 PUT с известной длиной.
4. Записать ready-метаданные и COMMIT PostgreSQL.
5. Только после успешного COMMIT вернуть 201.
Сбой DB после PUT или завершение процесса может оставить объект без строки БД.
Немедленного безусловного DELETE нет: при неопределённом результате COMMIT
он мог бы удалить уже подтверждённые данные. Вместо этого работает реальная
периодическая сверка: `src/cleanup.py`, сервис `cleanup` каждые 300 секунд.
Очистка обходит только prefix originals/, пропускает объекты моложе **одного часа**
по S3 LastModified. Для старого объекта берёт неблокирующий advisory lock,
повторно проверяет отсутствие строки Attachment и только затем удаляет объект.
Upload держит тот же lock от PUT до COMMIT. Занятый lock пропускается.
Недоступная БД не трактуется как «строки нет»: очистка прерывается без DELETE,
задача повторяется позже. Пагинация LIST ограничивает объём одного обходного блока.
Объекты с метаданными, даже пока не прикреплённые к сообщению, сохраняются.
Повторяемый ручной запуск: `scripts/cleanup.sh`; результат — счётчики scanned,
fresh, referenced, busy, deleted. Их можно сравнить до/после и повторить команду.
Проверены реальный rollback после PUT и SIGKILL во время INSERT после PUT.
В обоих случаях обнаружен фактический orphan и проверено его удаление.
Для проверки истечения grace часы **только тестового процесса cleanup** сдвинуты
на +2h: системные часы, S3 timestamps и production grace не менялись.
Реальное ожидание часа не выполнялось. Свежие файлы в обычном запуске остаются,
используемые файлы сохраняются и при сдвиге тестовых часов.
S3 bucket bootstrap: `mc mb --ignore-existing`, anonymous none, отдельный
пользователь API и policy только на один bucket/prefix. Root credentials
отсутствуют в API. Его GET/PUT/DELETE и LIST нужны для хранения и очистки;
управление пользователями/bucket policy не разрешено. Проверка вне API
дала 403 на anonymous GET существующего объекта и LIST bucket; обращение
пользователя API к bucket policy также дало 403. Повторные `up/start` действительно
повторяли bootstrap и миграции без потери данных.
## Реальные проверки
Итоговый полный прогон `scripts/verify.py` завершился с exit code 0.
Все 16 шагов перечислены в `evidence/verification.json`.
| Требование / проверка | Реальная команда / воздействие | Результат | Evidence |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Структура | make check | PASS | check.txt |
| OpenAPI | make validate PYTHON=.venv/bin/python | PASS | validate.txt |
| Доступный закреплённый образ | docker pull bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:… | PASS | image-pull.txt |
| Compose | docker compose --env-file .env config --quiet | PASS, exit 0 | compose-config.txt, verification.json |
| Сборка/миграции/bootstrap | docker compose --env-file .env up -d --build | PASS | build.txt |
| Весь публичный сценарий lab4 | make test BASE_URL=http://localhost:8084 | PASS | smoke.txt |
| Сессии/сообщения/файл после restart всех stores | make test-persistence … ACTION_SCRIPT=scripts/restart.sh | PASS | persistence.txt, restart-before.txt, restart-after.txt |
| Работа при отказе API | make test-failover … ACTION_SCRIPT=scripts/stop-one.sh | PASS; 116 успешных записей, 3 переходные ошибки; восстановление 2,280 с | failover.txt |
| Обе загрузившие реплики остановлены по очереди | python3 student-tests/attachment-failover.py | PASS, 2 сценария; SHA-256 совпал | attachment-failover.txt |
| PNG/JPEG, оригинал, доступ другого участника | python3 student-tests/negative.py | PASS | negative.txt |
| >10 MiB с Content-Length и chunked, >40M pixels | тот же negative.py | PASS, 413 | negative.txt |
| Битые PNG/JPEG, MIME/format/multipart | тот же negative.py | PASS, 415 | negative.txt |
| Filename, некорректные/повторные/лишние ID | тот же negative.py | PASS, 400 | negative.txt |
| Неизвестное/чужое/из другого чата вложение, auth | тот же negative.py | PASS, 404/401 | negative.txt |
| 10 конкурентных идемпотентных POST, default [] | тот же negative.py | PASS | negative.txt |
| Отказ S3 / PostgreSQL | python3 student-tests/faults.py, реальные stop/start | PASS, API JSON 503 с headers | faults.txt |
| DB failure после S3 PUT | trigger для одного случайного filename | PASS, orphan обнаружен и очищен | faults.txt |
| Обрыв upload | реальный TCP close до конца body | PASS, новых S3-объектов нет | faults.txt |
| SIGKILL после PUT до COMMIT | остановлены API, затем восстановлены | PASS, orphan обнаружен и очищен | faults.txt |
| Grace, сохранность referenced, повторная очистка | обычный и тестовый сдвинутый clock | PASS | faults.txt, cleanup.txt, cleanup-repeat.txt |
| Приватность и ограниченная policy | unsigned GET/LIST, API GetBucketPolicy | PASS: 403 / 403 / 403 | faults.txt |
| Все сохранённые оригиналы после проверок | python3 student-tests/audit-storage.py | PASS; 55 оригиналов, 157 371 407 bytes; каждый SHA-256 и размер | storage-audit.txt |
| 10 одновременных upload и RAM | python3 student-tests/upload-load.py | PASS, 10/10 | upload-load.json, upload-load.txt |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Все пути evidence в таблице относительны каталогу `evidence/`.
Negative suite — 16 групп, fault suite — 11. Failover содержит ожидаемые временные
ошибки при отключении реплики: они учтены, не скрыты; успешный итог означает
восстановление и сохранность подтверждённых записей, а не отсутствие сетевого отказа.
## Эксперименты и отказы
В разработке были неуспешные попытки: скачивание первоначальных тегов MinIO,
доступ sandbox к Docker/сети, а также неверная проверка cleanup в собственном
повторном fault-тесте. Последняя сравнивала со списком всех объектов, ошибочно
считая orphan от предыдущей аварии используемым. Исправлена именно проверка
сохранности на сравнение с ключами, реально указанными в PostgreSQL;
алгоритм очистки не ослаблялся. Исходный FAIL сохранён в `faults-initial.txt`.
В финальном прогоне FAIL нет. Публичные тесты не редактировались.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Эксперимент с памятью
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Запуск: `python3 student-tests/upload-load.py`, 2026-11-06 14:53:56 UTC.
Среда: macOS 26.6.2 arm64, Docker Linux aarch64, выделены 8 CPU и
8 218 316 800 байт RAM. API лимитированы каждый 0,75 CPU и 768 MiB.
Сервисы прогреты предыдущими функциональными проверками; отдельной серии
прогрева и повторов нагрузки не было.
## Ограничения и следующие шаги
Сгенерирован случайный RGB PNG 1600×1600, **7 684 008 bytes**. Один общий
файл отправлен 10 клиентами после barrier одновременно через HAProxy;
каждая реплика обрабатывает свои загрузки последовательно через semaphore.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
| Метрика | Измерено |
| --- | --- |
| Конкуренция / запросы | 10 / 10 |
| Успехи / ошибки | 10 / 0 |
| Длительность HTTP-пачки | 1,785141 с |
| Успешные запросы/с | 5,601799 |
| p50 / p95 / p99 успешных | 1,090735 / 1,743270 / 1,743270 с |
| API 1 sampled baseline / peak | 76,59 / **107,57 MiB** |
| API 2 sampled baseline / peak | 76,99 / **118,64 MiB** |
| Cgroup lifetime memory.peak API 1 / 2 | 131,89 / 134,75 MiB |
| CPU time API 1 / 2 за окно ~20 с | 2,439180 / 2,514936 CPU-с |
| Средний CPU относительно одного ядра за это окно | 12,19% / 12,57% |
Память измерена из cgroup v2 `memory.current` внутри каждого API-контейнера
каждые 20 мс, 907/905 samples; включает дочерний декодер, tmpfs и page cache.
Это наблюдённый sampled peak, не гарантия точного мгновенного максимума.
`memory.peak` относится ко всей жизни контейнера, включая прежнюю работу;
не выдаётся за пик исключительно этой нагрузки. Пики двух реплик могут
не совпадать по времени, складывать их в «пик стенда» нельзя.
CPU усреднён за окно наблюдения 20 с с простоем после короткой пачки,
поэтому не описывает CPU только активных 1,8 с.
Один локальный closed-loop запуск, накладные расходы клиента, малая выборка
и совместное использование host с другими лабораторными не позволяют
заявлять предельную пропускную способность или production SLA. Все исходные
числа и результаты каждого запроса — в `evidence/upload-load.json`.
+155
View File
@@ -0,0 +1,155 @@
name: mtusi2026lab4
services:
postgres:
image: postgres:16.8-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 20
cpus: 1.0
mem_limit: 512m
restart: unless-stopped
redis:
image: redis:7.4.2-alpine
command: [redis-server, --appendonly, 'yes', --appendfsync, always]
volumes: [redis_data:/data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 20
cpus: 0.25
mem_limit: 128m
restart: unless-stopped
migrate:
build: .
image: mtusi2026lab4-api:local
command: [python, -m, src.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
cpus: 0.5
mem_limit: 256m
restart: 'no'
api:
build: .
image: mtusi2026lab4-api:local
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
s3-bootstrap:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
deploy:
replicas: 2
healthcheck:
test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()']
interval: 2s
timeout: 3s
retries: 3
start_period: 5s
tmpfs:
- /tmp:size=16m,mode=1777
stop_grace_period: 6s
cpus: 0.75
mem_limit: 768m
restart: unless-stopped
cleanup:
image: mtusi2026lab4-api:local
command: [python, -m, src.cleanup_loop]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://s3:9000
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
depends_on:
api:
condition: service_healthy
healthcheck:
disable: true
restart: unless-stopped
cpus: 0.25
mem_limit: 256m
lb:
image: haproxy:3.0.9-alpine
ports: ['127.0.0.1:${PUBLIC_PORT:-8084}:8080']
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
depends_on:
api:
condition: service_healthy
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready']
interval: 5s
timeout: 3s
retries: 3
cpus: 0.5
mem_limit: 128m
restart: unless-stopped
s3:
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
entrypoint: [/opt/bitnami/minio/bin/minio]
user: '0:0'
command: [server, /data, --console-address, ':9001']
environment:
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
secrets: [s3_root_user, s3_root_password]
volumes: [s3_data:/data]
healthcheck:
test: [CMD, curl, -f, http://localhost:9000/minio/health/live]
interval: 2s
timeout: 2s
retries: 30
restart: unless-stopped
mem_limit: 768m
cpus: 1.0
s3-bootstrap:
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
entrypoint: [/bin/sh, /bootstrap.sh]
volumes:
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
- ./infra/s3-policy.json:/policy.json:ro
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
depends_on:
s3:
condition: service_healthy
restart: 'no'
volumes:
s3_data:
postgres_data:
redis_data:
secrets:
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
s3_root_user:
file: ./secrets/s3_root_user
s3_root_password:
file: ./secrets/s3_root_password
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+51
View File
@@ -0,0 +1,51 @@
CREATE TABLE IF NOT EXISTS users (
id UUID PRIMARY KEY,
username VARCHAR(32) NOT NULL UNIQUE
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name VARCHAR(100) NOT NULL
CHECK (length(btrim(display_name)) > 0),
password_hash TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chats (
id UUID PRIMARY KEY,
title VARCHAR(100) NOT NULL
CHECK (length(btrim(title)) > 0),
created_by UUID NOT NULL REFERENCES users(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id UUID NOT NULL REFERENCES users(id),
joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id UUID NOT NULL REFERENCES users(id),
text TEXT NOT NULL
CHECK (length(text) <= 4000 AND length(btrim(text)) > 0),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE TABLE IF NOT EXISTS sessions (
id UUID PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
token_hash TEXT NOT NULL UNIQUE,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ
);
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_chat_members_user_id
ON chat_members (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
ON sessions (expires_at);
+9
View File
@@ -0,0 +1,9 @@
CREATE TABLE message_requests (
chat_id UUID NOT NULL REFERENCES chats(id),
sender_id UUID NOT NULL REFERENCES users(id),
key VARCHAR(128) NOT NULL,
text TEXT NOT NULL,
message_id UUID REFERENCES messages(id),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
PRIMARY KEY (chat_id, sender_id, key)
);
+16
View File
@@ -0,0 +1,16 @@
CREATE TABLE attachments (
id UUID PRIMARY KEY,
chat_id UUID NOT NULL REFERENCES chats(id),
owner_id UUID NOT NULL REFERENCES users(id),
filename TEXT NOT NULL,
object_key TEXT NOT NULL UNIQUE,
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
sha256 CHAR(64) NOT NULL,
status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}';
ALTER TABLE messages ADD CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0));
+20
View File
@@ -0,0 +1,20 @@
{
"passed": true,
"both_uploaders_tested": 2,
"scenarios": [
{
"uploader_instance": "406afeb6-f014-4233-87fb-3ee2e5d0f599",
"stopped_container": "30fcbabd1e8c",
"survivor_instance": "8c155e11-6333-44b9-8520-b1e6e1e53485",
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"original_identical": true
},
{
"uploader_instance": "8c155e11-6333-44b9-8520-b1e6e1e53485",
"stopped_container": "050e1dff86f8",
"survivor_instance": "807def1e-c2d4-4555-896c-68b5796f7c8f",
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"original_identical": true
}
]
}
+166
View File
@@ -0,0 +1,166 @@
Compose now can delegate build to bake for better performances
Just set COMPOSE_BAKE=true
#0 building with "desktop-linux" instance using docker driver
#1 [migrate internal] load build definition from Dockerfile
#1 transferring dockerfile: 954B done
#1 DONE 0.0s
#2 [migrate internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
#2 DONE 0.0s
#3 [migrate internal] load .dockerignore
#3 transferring context: 222B done
#3 DONE 0.0s
#4 [migrate internal] load build context
#4 transferring context: 1.34kB done
#4 DONE 0.0s
#5 [migrate 1/8] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
#5 DONE 0.0s
#6 [migrate 2/8] WORKDIR /app
#6 CACHED
#7 [migrate 7/8] COPY --chown=appuser:appuser db/ ./db/
#7 CACHED
#8 [migrate 3/8] COPY requirements.txt .
#8 CACHED
#9 [migrate 5/8] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
#9 CACHED
#10 [migrate 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt
#10 CACHED
#11 [migrate 6/8] COPY --chown=appuser:appuser src/ ./src/
#11 CACHED
#12 [migrate 8/8] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
#12 CACHED
#13 [migrate] exporting to image
#13 exporting layers done
#13 exporting manifest sha256:87efe3e62f69573c004c4121ff309b1ad35bdf670075e413f766061b78b3efc5 done
#13 exporting config sha256:7be1ef706b170dc197efbeedfbd3d09501e6a5f4e8fd590df6b3e6ff14ebdb49 done
#13 exporting attestation manifest sha256:2e280e83286ae3dab79ddf06b9735524b172bbfdde5487869259e21b361ac693 done
#13 exporting manifest list sha256:26a6698cdea3199fe81ad434ade082fb14ad71f20b635254f0f8ed7c3cdcba5e done
#13 naming to docker.io/library/mtusi2026lab4-api:local done
#13 unpacking to docker.io/library/mtusi2026lab4-api:local done
#13 DONE 0.0s
#14 [migrate] resolving provenance for metadata file
#14 DONE 0.0s
#15 [api internal] load build definition from Dockerfile
#15 transferring dockerfile: 954B done
#15 DONE 0.0s
#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
#2 DONE 0.0s
#16 [api internal] load .dockerignore
#16 transferring context: 222B done
#16 DONE 0.0s
#17 [api internal] load build context
#17 transferring context: 1.34kB done
#17 DONE 0.0s
#5 [api 1/8] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
#5 DONE 0.0s
#18 [api 7/8] COPY --chown=appuser:appuser db/ ./db/
#18 CACHED
#6 [api 2/8] WORKDIR /app
#6 CACHED
#19 [api 6/8] COPY --chown=appuser:appuser src/ ./src/
#19 CACHED
#20 [api 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt
#20 CACHED
#21 [api 5/8] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
#21 CACHED
#22 [api 3/8] COPY requirements.txt .
#22 CACHED
#23 [api 8/8] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
#23 CACHED
#24 [api] exporting to image
#24 exporting layers done
#24 exporting manifest sha256:36e8cc71595fc04804ec665284eb6328bd087de9dded24bf5363e43219b05be3 done
#24 exporting config sha256:90e3dbc5cad2036701f73221a02ab439270f933b193dbd812901db095860b963 done
#24 exporting attestation manifest sha256:2364f47deb3b19f215c9813c211c0510a5c625fa838558d4fbc002e090e306e2 done
#24 exporting manifest list sha256:bf8b09641c0bf0889e4103bc4a7ac1dd6563feeda39b19b8762788eab50cbd5a done
#24 naming to docker.io/library/mtusi2026lab4-api:local done
#24 unpacking to docker.io/library/mtusi2026lab4-api:local done
#24 DONE 0.0s
#25 [api] resolving provenance for metadata file
#25 DONE 0.0s
api Built
migrate Built
Container mtusi2026lab4-redis-1 Running
Container mtusi2026lab4-s3-1 Running
Container mtusi2026lab4-postgres-1 Running
Container mtusi2026lab4-s3-bootstrap-1 Created
Container mtusi2026lab4-api-2 Stopping
Container mtusi2026lab4-api-1 Stopping
Container mtusi2026lab4-api-1 Stopped
Container mtusi2026lab4-api-2 Stopped
Container mtusi2026lab4-migrate-1 Recreate
Container mtusi2026lab4-migrate-1 Recreated
Container mtusi2026lab4-lb-1 Stopping
Container mtusi2026lab4-cleanup-1 Stopping
Container mtusi2026lab4-lb-1 Stopped
Container mtusi2026lab4-cleanup-1 Stopped
Container mtusi2026lab4-api-1 Recreate
Container mtusi2026lab4-lb-1 Stopping
Container mtusi2026lab4-cleanup-1 Stopping
Container mtusi2026lab4-cleanup-1 Stopped
Container mtusi2026lab4-lb-1 Stopped
Container mtusi2026lab4-api-2 Recreate
Container mtusi2026lab4-api-1 Recreated
Container mtusi2026lab4-api-2 Recreated
Container mtusi2026lab4-lb-1 Created
Container mtusi2026lab4-cleanup-1 Recreate
Container mtusi2026lab4-cleanup-1 Recreated
Container mtusi2026lab4-postgres-1 Waiting
Container mtusi2026lab4-s3-1 Waiting
Container mtusi2026lab4-s3-1 Healthy
Container mtusi2026lab4-s3-bootstrap-1 Starting
Container mtusi2026lab4-postgres-1 Healthy
Container mtusi2026lab4-migrate-1 Starting
Container mtusi2026lab4-s3-bootstrap-1 Started
Container mtusi2026lab4-migrate-1 Started
Container mtusi2026lab4-s3-bootstrap-1 Waiting
Container mtusi2026lab4-migrate-1 Waiting
Container mtusi2026lab4-redis-1 Waiting
Container mtusi2026lab4-redis-1 Healthy
Container mtusi2026lab4-s3-bootstrap-1 Exited
Container mtusi2026lab4-migrate-1 Exited
Container mtusi2026lab4-api-2 Starting
Container mtusi2026lab4-api-2 Started
Container mtusi2026lab4-api-1 Starting
Container mtusi2026lab4-api-1 Started
Container mtusi2026lab4-api-2 Waiting
Container mtusi2026lab4-api-1 Waiting
Container mtusi2026lab4-api-2 Waiting
Container mtusi2026lab4-api-1 Waiting
Container mtusi2026lab4-api-2 Healthy
Container mtusi2026lab4-api-1 Healthy
Container mtusi2026lab4-lb-1 Starting
Container mtusi2026lab4-api-2 Healthy
Container mtusi2026lab4-api-1 Healthy
Container mtusi2026lab4-cleanup-1 Starting
Container mtusi2026lab4-lb-1 Started
Container mtusi2026lab4-cleanup-1 Started
+77
View File
@@ -0,0 +1,77 @@
MODIFIED
.dockerignore
.env.example
.gitignore
REPORT.md
infra/README.md
ADDED (unstaged)
Dockerfile
compose.yaml
db/migrations/001_initial.sql
db/migrations/002_idempotency.sql
db/migrations/003_attachments.sql
evidence/attachment-failover.txt
evidence/build.txt
evidence/changed-files.txt
evidence/check.txt
evidence/cleanup-repeat.txt
evidence/cleanup.txt
evidence/compose-config.txt
evidence/development-notes.txt
evidence/failover.txt
evidence/faults-initial.txt
evidence/faults.txt
evidence/final-audit.txt
evidence/git-status.txt
evidence/image-pull.txt
evidence/negative.txt
evidence/persistence.txt
evidence/restart-after.txt
evidence/restart-before.txt
evidence/restore-after-failover.txt
evidence/smoke.txt
evidence/storage-audit.txt
evidence/storage-image-switch.txt
evidence/upload-load.json
evidence/upload-load.txt
evidence/validate.txt
evidence/verification.json
infra/haproxy.cfg
infra/s3-bootstrap.sh
infra/s3-policy.json
requirements.txt
scripts/README.md
scripts/cleanup.sh
scripts/compose.sh
scripts/init-local.py
scripts/restart.sh
scripts/stop-one.sh
scripts/verify.py
src/__init__.py
src/attachments.py
src/auth.py
src/chat_repository.py
src/cleanup.py
src/cleanup_loop.py
src/database.py
src/image_check.py
src/main.py
src/migrate.py
src/session_cache.py
src/storage.py
src/user_repository.py
student-tests/attachment-failover.py
student-tests/audit-storage.py
student-tests/faults.py
student-tests/fixtures/README.md
student-tests/fixtures/sample.jpg
student-tests/negative.py
student-tests/upload-load.py
LOCAL IGNORED
DEFENSE.md
.env
secrets/
backups/
.venv/
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+1
View File
@@ -0,0 +1 @@
{"scanned": 55, "fresh": 55, "referenced": 0, "busy": 0, "deleted": 0}
+1
View File
@@ -0,0 +1 @@
{"scanned": 55, "fresh": 55, "referenced": 0, "busy": 0, "deleted": 0}
View File
+13
View File
@@ -0,0 +1,13 @@
Initial environment/development failures (not final PASS):
- pip install in sandbox: DNS/host lookup denied. Retried with network approval; installed successfully.
- Docker Desktop credential helper absent from PATH. scripts/compose.sh adds the existing Docker Desktop bin path.
- minio/minio release-tag pull: access denied. Quay release-tag and cached working digest pull: 401.
- A working cached Quay image was first used for development. Final deployment changed to accessible
bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20.
Docker pull succeeded. Before switching, a stopped-volume tar backup was saved locally under ignored backups/.
All 33 previously committed originals (79300697 bytes) retained matching sizes and SHA-256 after switch.
- Own repeated cleanup test incorrectly treated all pre-existing objects (including an orphan from an earlier
crash) as referenced. Failure retained in faults-initial.txt. Assertion corrected to compare only keys backed
by committed PostgreSQL metadata. Cleanup algorithm not weakened; final suite rerun.
- Sandbox Docker socket access failures were retried with the required execution permissions.
Public tests and mandatory contracts were not edited. No Git staging/commit/push/PR actions performed.
+4
View File
@@ -0,0 +1,4 @@
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8084"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 3, "first_recovery_seconds": 2.28}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+10
View File
@@ -0,0 +1,10 @@
Traceback (most recent call last):
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/student-tests/faults.py", line 204, in <module>
main()
~~~~^^
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/student-tests/faults.py", line 92, in main
check(not orphan & final and before <= final, 'cleanup lost used data')
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/tests/client.py", line 24, in check
raise Failure(message)
client.Failure: cleanup lost used data
+29
View File
@@ -0,0 +1,29 @@
{
"passed": [
"S3_PUT_then_DB_failure_returns_503_with_real_orphan",
"fresh_orphan_survives_one_hour_grace",
"expired_orphan_deleted_referenced_objects_preserved_test_clock",
"cleanup_repeatable",
"anonymous_GET_and_LIST_denied_API_cannot_read_policy",
"s3_outage_API_503",
"postgres_outage_API_503",
"interrupted_upload_no_S3_object",
"API_SIGKILL_after_PUT_before_COMMIT_orphan_recoverable",
"API_crash_orphan_actually_cleaned_after_test_clock_grace",
"confirmed_original_survives_faults"
],
"count": 11,
"privacy_status_codes": [
403,
403,
403
],
"cleanup_expired_result": {
"scanned": 41,
"fresh": 0,
"referenced": 40,
"busy": 0,
"deleted": 1
},
"grace_test": "test process clock +2h; production grace remains 1h"
}
+21
View File
@@ -0,0 +1,21 @@
$ git diff --check
exit_code=0
$ git diff --exit-code -- tests contracts README.md COURSE.md CONTRIBUTING.md lab.json
exit_code=0
$ git diff --cached --exit-code
exit_code=0
$ git check-ignore DEFENSE.md .env secrets/postgres_password backups/minio-before-image-change.tar
exit_code=0
DEFENSE.md
.env
secrets/postgres_password
backups/minio-before-image-change.tar
$ git -C ../mtusi2026lab3 status -sb
exit_code=0
## solution/BPI2301-Dugina...origin/solution/BPI2301-Dugina
PASS: 16 verification steps have actual exit_code=0; Python syntax parsed; local password values absent from tracked/nonignored files.
+48
View File
@@ -0,0 +1,48 @@
## solution/BPI2301-Dugina
M .dockerignore
M .env.example
M .gitignore
M REPORT.md
M infra/README.md
?? Dockerfile
?? compose.yaml
?? db/
?? evidence/attachment-failover.txt
?? evidence/build.txt
?? evidence/changed-files.txt
?? evidence/check.txt
?? evidence/cleanup-repeat.txt
?? evidence/cleanup.txt
?? evidence/compose-config.txt
?? evidence/development-notes.txt
?? evidence/failover.txt
?? evidence/faults-initial.txt
?? evidence/faults.txt
?? evidence/final-audit.txt
?? evidence/git-status.txt
?? evidence/image-pull.txt
?? evidence/negative.txt
?? evidence/persistence.txt
?? evidence/restart-after.txt
?? evidence/restart-before.txt
?? evidence/restore-after-failover.txt
?? evidence/smoke.txt
?? evidence/storage-audit.txt
?? evidence/storage-image-switch.txt
?? evidence/upload-load.json
?? evidence/upload-load.txt
?? evidence/validate.txt
?? evidence/verification.json
?? infra/haproxy.cfg
?? infra/s3-bootstrap.sh
?? infra/s3-policy.json
?? requirements.txt
?? scripts/README.md
?? scripts/cleanup.sh
?? scripts/compose.sh
?? scripts/init-local.py
?? scripts/restart.sh
?? scripts/stop-one.sh
?? scripts/verify.py
?? src/
?? student-tests/
+4
View File
@@ -0,0 +1,4 @@
docker.io/bitnamilegacy/minio@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20: Pulling from bitnamilegacy/minio
Digest: sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
Status: Image is up to date for bitnamilegacy/minio@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
docker.io/bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
+22
View File
@@ -0,0 +1,22 @@
{
"passed": [
"bytes_over_10MiB",
"pixels_over_40M",
"broken_png",
"broken_jpeg",
"mime_mismatch",
"invalid_format",
"multipart_rejected",
"dangerous_filenames",
"original_checksum_and_shared_access",
"JPEG_success_and_checksum",
"download_auth_and_acl",
"unknown_foreign_and_other_chat_attachment",
"attachment_id_validation",
"attachment_idempotency_across_replicas",
"chunked_body_limit",
"client_disconnect_server_survives"
],
"count": 16,
"note": "Fault/orphan checks are separate in faults.py"
}
+3
View File
@@ -0,0 +1,3 @@
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8084"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+7
View File
@@ -0,0 +1,7 @@
/mtusi2026lab4-api-1 2026-11-06T14:52:10.90533168Z running
/mtusi2026lab4-api-2 2026-11-06T14:52:10.803373597Z running
/mtusi2026lab4-cleanup-1 2026-11-06T14:52:20.417700921Z running
/mtusi2026lab4-lb-1 2026-11-06T14:52:10.491475222Z running
/mtusi2026lab4-postgres-1 2026-11-06T14:52:10.469402597Z running
/mtusi2026lab4-redis-1 2026-11-06T14:52:10.394762722Z running
/mtusi2026lab4-s3-1 2026-11-06T14:52:10.447334638Z running
+7
View File
@@ -0,0 +1,7 @@
/mtusi2026lab4-api-1 2026-11-06T14:52:01.854436551Z running
/mtusi2026lab4-api-2 2026-11-06T14:52:01.779041218Z running
/mtusi2026lab4-cleanup-1 2026-11-06T14:52:07.48552572Z running
/mtusi2026lab4-lb-1 2026-11-06T14:52:07.484992512Z running
/mtusi2026lab4-postgres-1 2026-11-06T14:44:12.610996625Z running
/mtusi2026lab4-redis-1 2026-11-06T14:43:25.264236673Z running
/mtusi2026lab4-s3-1 2026-11-06T14:50:28.060451049Z running
+16
View File
@@ -0,0 +1,16 @@
Container mtusi2026lab4-s3-1 Waiting
Container mtusi2026lab4-postgres-1 Waiting
Container mtusi2026lab4-postgres-1 Healthy
Container mtusi2026lab4-migrate-1 Starting
Container mtusi2026lab4-s3-1 Healthy
Container mtusi2026lab4-s3-bootstrap-1 Starting
Container mtusi2026lab4-s3-bootstrap-1 Started
Container mtusi2026lab4-migrate-1 Started
Container mtusi2026lab4-s3-bootstrap-1 Waiting
Container mtusi2026lab4-migrate-1 Waiting
Container mtusi2026lab4-redis-1 Waiting
Container mtusi2026lab4-s3-bootstrap-1 Exited
Container mtusi2026lab4-redis-1 Healthy
Container mtusi2026lab4-migrate-1 Exited
Container mtusi2026lab4-api-2 Starting
Container mtusi2026lab4-api-2 Started
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8084"
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+5
View File
@@ -0,0 +1,5 @@
{
"verified_originals": 55,
"verified_bytes": 157371407,
"all_checksums_match": true
}
+5
View File
@@ -0,0 +1,5 @@
{
"verified_originals": 33,
"verified_bytes": 79300697,
"all_checksums_match": true
}
+95
View File
@@ -0,0 +1,95 @@
{
"utc": "2026-11-06T14:53:56Z",
"concurrency": 10,
"requests": 10,
"original_bytes": 7684008,
"dimensions": [
1600,
1600
],
"successful_requests": 10,
"failed_requests": 0,
"elapsed_seconds": 1.785140791995218,
"successful_rps": 5.601799054080877,
"latency_success_seconds": {
"p50": 1.0907350000052247,
"p95": 1.743270292005036,
"p99": 1.743270292005036
},
"api_cgroups": [
{
"sample_count": 907,
"sampled_baseline_bytes": 80306176,
"sampled_peak_bytes": 112791552,
"cgroup_lifetime_peak_bytes": 138301440,
"elapsed_seconds": 20.002547300000515,
"cpu_usage_seconds": 2.43918,
"container": "050e1dff86f8",
"average_cpu_percent_one_core": 12.194346867010967
},
{
"sample_count": 905,
"sampled_baseline_bytes": 80728064,
"sampled_peak_bytes": 124407808,
"cgroup_lifetime_peak_bytes": 141295616,
"elapsed_seconds": 20.01476109199939,
"cpu_usage_seconds": 2.514936,
"container": "30fcbabd1e8c",
"average_cpu_percent_one_core": 12.565406044268544
}
],
"requests_detail": [
{
"ok": true,
"seconds": 0.8693261250155047,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.4904946660099085,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.4210281249834225,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.266302750009345,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 0.7139787090127356,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.743270292005036,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 0.6961546249804087,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 0.9460117089911364,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.0907350000052247,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.2004604999965522,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
}
],
"limitations": "One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.",
"docker_host": "aarch64 CPUs=8 RAM=8218316800"
}
+95
View File
@@ -0,0 +1,95 @@
{
"utc": "2026-11-06T14:53:56Z",
"concurrency": 10,
"requests": 10,
"original_bytes": 7684008,
"dimensions": [
1600,
1600
],
"successful_requests": 10,
"failed_requests": 0,
"elapsed_seconds": 1.785140791995218,
"successful_rps": 5.601799054080877,
"latency_success_seconds": {
"p50": 1.0907350000052247,
"p95": 1.743270292005036,
"p99": 1.743270292005036
},
"api_cgroups": [
{
"sample_count": 907,
"sampled_baseline_bytes": 80306176,
"sampled_peak_bytes": 112791552,
"cgroup_lifetime_peak_bytes": 138301440,
"elapsed_seconds": 20.002547300000515,
"cpu_usage_seconds": 2.43918,
"container": "050e1dff86f8",
"average_cpu_percent_one_core": 12.194346867010967
},
{
"sample_count": 905,
"sampled_baseline_bytes": 80728064,
"sampled_peak_bytes": 124407808,
"cgroup_lifetime_peak_bytes": 141295616,
"elapsed_seconds": 20.01476109199939,
"cpu_usage_seconds": 2.514936,
"container": "30fcbabd1e8c",
"average_cpu_percent_one_core": 12.565406044268544
}
],
"requests_detail": [
{
"ok": true,
"seconds": 0.8693261250155047,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.4904946660099085,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.4210281249834225,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.266302750009345,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 0.7139787090127356,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.743270292005036,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 0.6961546249804087,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 0.9460117089911364,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
},
{
"ok": true,
"seconds": 1.0907350000052247,
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
},
{
"ok": true,
"seconds": 1.2004604999965522,
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
}
],
"limitations": "One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.",
"docker_host": "aarch64 CPUs=8 RAM=8218316800"
}
+2
View File
@@ -0,0 +1,2 @@
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+164
View File
@@ -0,0 +1,164 @@
[
{
"check": "check",
"command": [
"make",
"check"
],
"exit_code": 0,
"seconds": 0.074
},
{
"check": "validate",
"command": [
"make",
"validate",
"PYTHON=.venv/bin/python"
],
"exit_code": 0,
"seconds": 0.546
},
{
"check": "image-pull",
"command": [
"docker",
"pull",
"bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20"
],
"exit_code": 0,
"seconds": 17.858
},
{
"check": "compose-config",
"command": [
"docker",
"compose",
"--env-file",
".env",
"config",
"--quiet"
],
"exit_code": 0,
"seconds": 0.092
},
{
"check": "build",
"command": [
"docker",
"compose",
"--env-file",
".env",
"up",
"-d",
"--build"
],
"exit_code": 0,
"seconds": 19.796
},
{
"check": "smoke",
"command": [
"make",
"test",
"BASE_URL=http://localhost:8084"
],
"exit_code": 0,
"seconds": 1.332
},
{
"check": "persistence",
"command": [
"make",
"test-persistence",
"BASE_URL=http://localhost:8084",
"ACTION_SCRIPT=scripts/restart.sh"
],
"exit_code": 0,
"seconds": 11.799
},
{
"check": "failover",
"command": [
"make",
"test-failover",
"BASE_URL=http://localhost:8084",
"ACTION_SCRIPT=scripts/stop-one.sh"
],
"exit_code": 0,
"seconds": 16.266
},
{
"check": "restore-after-failover",
"command": [
"docker",
"compose",
"--env-file",
".env",
"start",
"api"
],
"exit_code": 0,
"seconds": 1.842
},
{
"check": "negative",
"command": [
"/opt/homebrew/opt/python@3.13/bin/python3.13",
"student-tests/negative.py"
],
"exit_code": 0,
"seconds": 3.637
},
{
"check": "faults",
"command": [
"/opt/homebrew/opt/python@3.13/bin/python3.13",
"student-tests/faults.py"
],
"exit_code": 0,
"seconds": 24.98
},
{
"check": "attachment-failover",
"command": [
"/opt/homebrew/opt/python@3.13/bin/python3.13",
"student-tests/attachment-failover.py"
],
"exit_code": 0,
"seconds": 19.642
},
{
"check": "upload-load",
"command": [
"/opt/homebrew/opt/python@3.13/bin/python3.13",
"student-tests/upload-load.py"
],
"exit_code": 0,
"seconds": 21.236
},
{
"check": "cleanup",
"command": [
"scripts/cleanup.sh"
],
"exit_code": 0,
"seconds": 1.036
},
{
"check": "cleanup-repeat",
"command": [
"scripts/cleanup.sh"
],
"exit_code": 0,
"seconds": 0.838
},
{
"check": "storage-audit",
"command": [
"/opt/homebrew/opt/python@3.13/bin/python3.13",
"student-tests/audit-storage.py"
],
"exit_code": 0,
"seconds": 1.368
}
]
+32 -4
View File
@@ -1,7 +1,35 @@
# Инфраструктура студента
# Стенд lab4
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
Compose project `mtusi2026lab4`, внешний вход `127.0.0.1:8084`.
Все остальные сервисы доступны только в Docker-сети этого проекта.
API-порты, S3 и консоль MinIO 9001 не публикуются.
Lab3 на 8080 и его volumes не используются.
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
- `lb`: HAProxy, round-robin, проверки `/health/ready`, без автоматического replay POST.
- `api`: две реплики FastAPI без persistent volume; `/tmp` — tmpfs 16 MiB.
- `postgres`: пользователи, сессии, чаты, сообщения, идемпотентность, Attachment.
- `redis`: только кеш профилей; проверка сессии всегда идёт в PostgreSQL.
- `s3`: MinIO, оригиналы в отдельном persistent volume `s3_data`.
- `migrate`: последовательные SQL-миграции под advisory lock, таблица schema_migrations.
- `s3-bootstrap`: mc из закреплённого Bitnami-образа, идемпотентный bucket,
запрет anonymous доступа, отдельный пользователь API и prefix-scoped policy.
- `cleanup`: каждые 300 секунд удаляет только orphan старше часа;
повтор вручную: `scripts/cleanup.sh`.
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
`python3 scripts/init-local.py` генерирует локальные secrets и не перезаписывает
существующие значения. Реальные значения исключены из Git и Docker build context.
API не получает root credentials MinIO. `s3-policy.json` разрешает только
PUT/GET/DELETE `originals/*` и LIST этого prefix одного bucket. DELETE и LIST
используются механизмом очистки; создание bucket/управление policy запрещены API.
Root credentials есть только у S3 и одноразового bootstrap.
Health: PostgreSQL pg_isready, Redis ping, MinIO /minio/health/live,
API — запрос к PostgreSQL + минимальный LIST S3; LB — API readiness.
Cleanup — периодическая CLI-задача с рестартом при завершении процесса,
её результат виден в `docker compose logs cleanup`.
При потере S3 обе API перестают быть ready: LB может вернуть собственный 503.
Прямое обращение к API в fault-тесте отдельно подтверждает правильный JSON 503.
Безопасная остановка: `scripts/compose.sh stop`.
Возобновление: `scripts/compose.sh up -d`.
Volumes не удаляются ни одним служебным скриптом.
+35
View File
@@ -0,0 +1,35 @@
global
log stdout format raw local0
maxconn 1024
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 60s
timeout server 60s
timeout check 1s
# Never replay a potentially committed POST automatically.
retries 0
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold obsolete 1s
hold nx 1s
frontend public
bind :8080
default_backend replicas
backend replicas
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 1-10 api:8080 check
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments
mc anonymous set none local/messenger-attachments
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local attachments-api /policy.json
mc admin policy attach local attachments-api --user "$(cat /run/secrets/s3_access_key_id)"
+32
View File
@@ -0,0 +1,32 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::messenger-attachments/originals/*"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"originals/*"
]
}
}
}
]
}
+9
View File
@@ -0,0 +1,9 @@
fastapi==0.115.12
uvicorn==0.34.2
pydantic==2.11.4
psycopg[binary]==3.2.6
psycopg-pool==3.2.6
redis==5.2.1
argon2-cffi==23.1.0
boto3==1.38.12
Pillow==11.2.1
+41
View File
@@ -0,0 +1,41 @@
# Команды оператора
Из корня lab4:
```sh
python3 scripts/init-local.py
scripts/compose.sh up -d --build
scripts/compose.sh ps
BASE_URL=http://localhost:8084 make test
make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
scripts/compose.sh start api
scripts/cleanup.sh
```
`compose.sh` добавляет Docker Desktop credential helper в PATH на macOS.
`restart.sh` перезапускает все хранилища/API/LB/cleanup без удаления volumes;
сохраняет времена запуска до/после в evidence/restart-{before,after}.txt.
`stop-one.sh` сопоставляет TARGET_INSTANCE_ID ответу локального API внутри
контейнера и останавливает только найденную реплику. Возврат 1 означает,
что реплика не найдена; восстановление отдельно: `scripts/compose.sh start api`.
Полный проверочный прогон: `python3 scripts/verify.py`. Он намеренно выполняет
отказы и перезапуски только проекта lab4, создаёт синтетические данные,
сохраняет реальные exit codes/выводы в evidence. Пользовательские данные не удаляет.
Отдельные проверки (BASE_URL по умолчанию localhost:8084):
```sh
python3 student-tests/negative.py
python3 student-tests/faults.py
python3 student-tests/attachment-failover.py
python3 student-tests/upload-load.py
```
`faults.py` временно создаёт DB trigger для единственного случайного имени файла,
останавливает S3/БД, завершает API после PUT, проверяет очистку и восстанавливает
сервисы в finally. Проверка истечения grace использует часы тестового процесса +2h:
не меняет системное время, production-код grace или timestamps объектов.
Не запускайте fault/нагрузочные проверки одновременно друг с другом.
Проверки предназначены для отдельного локального учебного стенда.
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
./scripts/compose.sh exec -T api python -m src.cleanup
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
fi
exec docker compose --env-file .env "$@"
+21
View File
@@ -0,0 +1,21 @@
"""Generate local credentials once; never overwrite an existing secret."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
directory = root / 'secrets'
directory.mkdir(mode=0o700, exist_ok=True)
values = {'postgres_password': secrets.token_hex(24), 's3_root_user': 'lab4root', 's3_root_password': secrets.token_hex(24), 's3_access_key_id': 'lab4api', 's3_secret_access_key': secrets.token_hex(24)}
for name, value in values.items():
path = directory / name
if not path.exists():
path.write_text(value + '\n')
path.chmod(0o600)
path = directory / 'database_url'
if not path.exists():
password = (directory / 'postgres_password').read_text().strip()
path.write_text(f'postgresql://messenger:{password}@postgres:5432/messenger\n')
path.chmod(0o600)
path = root / '.env'
if not path.exists():
path.write_text('COMPOSE_PROJECT_NAME=mtusi2026lab4\nPUBLIC_PORT=8084\n')
print('Local configuration ready (secret values suppressed)')
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
mkdir -p evidence
containers=$(./scripts/compose.sh ps -q postgres redis s3 api lb cleanup)
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-before.txt
./scripts/compose.sh restart postgres redis s3 api lb cleanup
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-after.txt
+12
View File
@@ -0,0 +1,12 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}"
for container in $(./scripts/compose.sh ps -q api); do
instance=$(docker exec "$container" python -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])')
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop -t 2 "$container" >/dev/null
exit 0
fi
done
exit 1
+49
View File
@@ -0,0 +1,49 @@
"""Explicit full lab4 validation; includes lab4-only restarts and fault injection."""
import json
import os
from pathlib import Path
import subprocess
import sys
import time
root = Path(__file__).resolve().parents[1]
os.chdir(root)
env = dict(os.environ, BASE_URL='http://localhost:8084')
env['PATH'] = '/Applications/Docker.app/Contents/Resources/bin:' + env.get('PATH', '')
compose = ['docker', 'compose', '--env-file', '.env']
checks = [
('check', ['make', 'check']),
('validate', ['make', 'validate', 'PYTHON=.venv/bin/python']),
('image-pull', ['docker', 'pull', 'bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20']),
('compose-config', compose + ['config', '--quiet']),
('build', compose + ['up', '-d', '--build']),
('smoke', ['make', 'test', 'BASE_URL=http://localhost:8084']),
('persistence', ['make', 'test-persistence', 'BASE_URL=http://localhost:8084', 'ACTION_SCRIPT=scripts/restart.sh']),
('failover', ['make', 'test-failover', 'BASE_URL=http://localhost:8084', 'ACTION_SCRIPT=scripts/stop-one.sh']),
('restore-after-failover', compose + ['start', 'api']),
('negative', [sys.executable, 'student-tests/negative.py']),
('faults', [sys.executable, 'student-tests/faults.py']),
('attachment-failover', [sys.executable, 'student-tests/attachment-failover.py']),
('upload-load', [sys.executable, 'student-tests/upload-load.py']),
('cleanup', ['scripts/cleanup.sh']),
('cleanup-repeat', ['scripts/cleanup.sh']),
('storage-audit', [sys.executable, 'student-tests/audit-storage.py']),
]
results = []
try:
for name, command in checks:
print('RUN:', name, flush=True)
start = time.monotonic()
with (root / 'evidence' / (name + '.txt')).open('w') as output:
result = subprocess.run(command, env=env, stdout=output, stderr=subprocess.STDOUT)
record = {'check': name, 'command': command, 'exit_code': result.returncode, 'seconds': round(time.monotonic()-start, 3)}
results.append(record)
(root / 'evidence/verification.json').write_text(json.dumps(results, indent=2) + '\n')
print(('PASS:' if result.returncode == 0 else 'FAIL:'), name, 'seconds=', record['seconds'], flush=True)
if result.returncode:
print((root / 'evidence' / (name + '.txt')).read_text(), flush=True)
raise SystemExit(result.returncode)
if name in ('restore-after-failover', 'attachment-failover'):
time.sleep(4)
finally:
subprocess.run(compose + ['start', 'postgres', 's3', 'api'], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
View File
+137
View File
@@ -0,0 +1,137 @@
import asyncio
import hashlib
import subprocess
import sys
import tempfile
import unicodedata
from uuid import UUID, uuid4
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import HTTPException, Request
from fastapi.responses import StreamingResponse
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from .database import get_connection
from .storage import s3, BUCKET
MAX_BYTES = 10 * 1024**2
upload_slots = asyncio.Semaphore(1)
def serialize(row):
result = {key: str(row[key]) for key in ('id', 'chat_id', 'owner_id')}
result.update({key: row[key] for key in ('filename', 'content_type', 'size_bytes', 'sha256', 'status')})
result.update({key: row[key].isoformat().replace('+00:00', 'Z') for key in ('created_at', 'updated_at')})
return dict(result, variants=[], error_code=None)
def persist(path, chat, owner, filename, mime, size, checksum):
attachment_id = uuid4()
key = 'originals/' + str(attachment_id)
# The same lock is taken by cleanup before its final DB check/deletion.
# A process crash releases the transaction; an uncommitted object is later cleaned.
with get_connection() as conn:
with conn.transaction():
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (key,))
with open(path, 'rb') as source:
s3.put_object(Bucket=BUCKET, Key=key, Body=source, ContentLength=size, ContentType=mime)
row = conn.execute('''INSERT INTO attachments
(id, chat_id, owner_id, filename, object_key, size_bytes, content_type, sha256)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *''',
(attachment_id, UUID(chat), UUID(owner), filename, key, size, mime, checksum)).fetchone()
# Exiting transaction above commits before returning 201.
return serialize(row)
def accessible(attachment_id, user_id):
try:
aid = UUID(attachment_id)
except ValueError:
raise HTTPException(400, 'Неверный UUID') from None
with get_connection() as conn:
row = conn.execute('''SELECT a.* FROM attachments a
JOIN chat_members cm ON cm.chat_id=a.chat_id
WHERE a.id=%s AND cm.user_id=%s''', (aid, UUID(user_id))).fetchone()
if row is None:
raise HTTPException(404, 'Вложение не найдено')
return row
def register(app, current_user, get_chat):
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
async def upload(chat_id: str, request: Request):
user = await run_in_threadpool(current_user, request)
chat = await run_in_threadpool(get_chat, chat_id, request)
filename = request.query_params.get('filename', '')
if (not 1 <= len(filename) <= 255 or not filename.strip()
or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename)):
raise HTTPException(400, 'Недопустимое имя файла')
mime = request.headers.get('content-type', '').split(';')[0].strip().lower()
if mime not in ('image/png', 'image/jpeg'):
raise HTTPException(415, 'Ожидается PNG или JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isascii() or not length.isdecimal() or len(length) > 12:
raise HTTPException(400, 'Неверный размер')
if int(length) > MAX_BYTES:
raise HTTPException(413, 'Файл больше 10 MiB')
try:
await asyncio.wait_for(upload_slots.acquire(), timeout=30)
except asyncio.TimeoutError:
raise HTTPException(503, 'Загрузчик занят') from None
try:
with tempfile.NamedTemporaryFile(prefix='lab4-') as target:
size = 0
digest = hashlib.sha256()
try:
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise HTTPException(413, 'Файл больше 10 MiB')
target.write(chunk)
digest.update(chunk)
except ClientDisconnect:
raise HTTPException(400, 'Загрузка прервана') from None
except TimeoutError:
raise HTTPException(400, 'Истекло время загрузки') from None
target.flush()
def validate():
try:
return subprocess.run([sys.executable, '-m', 'src.image_check', target.name, mime],
timeout=10, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
except subprocess.TimeoutExpired:
return 15
code = await run_in_threadpool(validate)
if code != 0:
raise HTTPException(413 if code == 13 else 415, 'Недопустимое изображение')
return await run_in_threadpool(persist, target.name, chat['id'], user['id'], filename, mime, size, digest.hexdigest())
finally:
upload_slots.release()
@app.get('/api/v1/attachments/{attachment_id}')
def metadata(attachment_id: str, request: Request):
return serialize(accessible(attachment_id, current_user(request)['id']))
@app.get('/api/v1/attachments/{attachment_id}/content')
def content(attachment_id: str, request: Request):
row = accessible(attachment_id, current_user(request)['id'])
if request.query_params.get('variant', 'original') != 'original':
raise HTTPException(404, 'Вариант не найден')
obj = s3.get_object(Bucket=BUCKET, Key=row['object_key'])
def chunks():
try:
for chunk in obj['Body'].iter_chunks(64 * 1024):
yield chunk
finally:
obj['Body'].close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
'Content-Disposition': 'attachment', 'Cache-Control': 'private, no-store'})
@app.exception_handler(BotoCoreError)
@app.exception_handler(ClientError)
async def storage_error(request, exc):
from .main import error_response
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
+91
View File
@@ -0,0 +1,91 @@
import base64
import binascii
import hashlib
import os
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerifyMismatchError
password_hasher = PasswordHasher(
time_cost=3,
memory_cost=65536,
parallelism=4,
)
def hash_password(password: str) -> str:
return password_hasher.hash(password)
def verify_password(password_hash: str, password: str) -> bool:
try:
return password_hasher.verify(password_hash, password)
except (VerifyMismatchError, InvalidHashError):
return False
def parse_basic(authorization: str | None) -> tuple[str, str] | None:
if not authorization:
return None
scheme, separator, credentials = authorization.partition(" ")
if not separator or scheme.lower() != "basic":
return None
try:
raw = base64.b64decode(credentials, validate=True)
decoded = raw.decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError, binascii.Error):
return None
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
return None
if not 12 <= len(password) <= 128:
return None
return username, password
def parse_bearer(authorization: str | None) -> str | None:
if not authorization:
return None
scheme, separator, token = authorization.partition(" ")
if not separator or scheme.lower() != "bearer":
return None
if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512:
return None
return token
def create_token() -> str:
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
return hashlib.sha256(token.encode("ascii")).hexdigest()
def session_expiration() -> datetime:
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
if ttl < 1:
raise ValueError("SESSION_TTL_SECONDS must be positive")
return datetime.now(timezone.utc) + timedelta(seconds=ttl)
def format_datetime(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(
timespec="microseconds"
).replace("+00:00", "Z")
+245
View File
@@ -0,0 +1,245 @@
from uuid import UUID, uuid4
from .database import get_connection
def serialize_chat(row):
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": [str(member_id) for member_id in row["member_ids"]],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def serialize_message(row):
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"attachment_ids": [str(a) for a in row["attachment_ids"]],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def create_chat(creator_id, title, member_ids):
chat_id = uuid4()
members = [UUID(member_id) for member_id in member_ids]
with get_connection() as conn:
with conn.transaction():
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM users
WHERE id = ANY(%s::uuid[])
""",
(members,),
)
existing = {row["id"] for row in cur.fetchall()}
if len(existing) != len(members):
return None
cur.execute(
"""
INSERT INTO chats (id, title, created_by)
VALUES (%s, %s, %s)
RETURNING id, title, created_at
""",
(chat_id, title, UUID(creator_id)),
)
chat = cur.fetchone()
cur.executemany(
"""
INSERT INTO chat_members (chat_id, user_id)
VALUES (%s, %s)
""",
[(chat_id, member_id) for member_id in members],
)
return {
"id": str(chat["id"]),
"title": chat["title"],
"member_ids": member_ids,
"created_at": chat["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def get_chat(chat_id, user_id):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
c.id,
c.title,
c.created_at,
ARRAY(
SELECT cm2.user_id
FROM chat_members cm2
WHERE cm2.chat_id = c.id
ORDER BY cm2.user_id
) AS member_ids
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE c.id = %s
AND cm.user_id = %s
""",
(UUID(chat_id), UUID(user_id)),
)
row = cur.fetchone()
return serialize_chat(row) if row else None
def list_chats(user_id, limit, last_chat_id=None):
with get_connection() as conn:
with conn.cursor() as cur:
last_key = None
if last_chat_id is not None:
cur.execute(
"""
SELECT c.created_at, c.id
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE c.id = %s
AND cm.user_id = %s
""",
(UUID(last_chat_id), UUID(user_id)),
)
last_key = cur.fetchone()
if last_key is None:
return None
cur.execute(
"""
SELECT
c.id,
c.title,
c.created_at,
ARRAY(
SELECT cm2.user_id
FROM chat_members cm2
WHERE cm2.chat_id = c.id
ORDER BY cm2.user_id
) AS member_ids
FROM chats c
JOIN chat_members cm
ON cm.chat_id = c.id
WHERE cm.user_id = %s
AND (
%s::timestamptz IS NULL
OR (c.created_at, c.id) >
(%s::timestamptz, %s::uuid)
)
ORDER BY c.created_at, c.id
LIMIT %s
""",
(
UUID(user_id),
last_key["created_at"] if last_key else None,
last_key["created_at"] if last_key else None,
last_key["id"] if last_key else None,
limit + 1,
),
)
rows = cur.fetchall()
return [serialize_chat(row) for row in rows]
def create_message(chat_id, sender_id, text, key, attachment_ids):
# INSERT ON CONFLICT waits for a concurrent transaction on the same key.
# READ COMMITTED's next statement sees its committed result. Both the
# message and deduplication record commit together, or both roll back.
with get_connection() as conn:
with conn.transaction():
attachments = [UUID(a) for a in attachment_ids]
rows = conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])', (UUID(chat_id), attachments)).fetchall()
if len(rows) != len(attachments):
return None
scope = (UUID(chat_id), UUID(sender_id), key)
row = conn.execute(
"""INSERT INTO message_requests(chat_id, sender_id, key, text)
VALUES (%s, %s, %s, %s)
ON CONFLICT (chat_id, sender_id, key) DO NOTHING
RETURNING key""", (*scope, text)
).fetchone()
if row is None:
previous = conn.execute(
"""SELECT m.* FROM message_requests r
JOIN messages m ON m.id = r.message_id
WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""",
scope,
).fetchone()
if previous["text"] != text or previous["attachment_ids"] != attachments:
return "conflict"
return serialize_message(previous)
message = conn.execute(
"""INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids)
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
(uuid4(), UUID(chat_id), UUID(sender_id), text, attachments),
).fetchone()
conn.execute(
"""UPDATE message_requests SET message_id=%s
WHERE chat_id=%s AND sender_id=%s AND key=%s""",
(message["id"], *scope),
)
return serialize_message(message)
def list_messages(chat_id, limit, last_message_id=None):
with get_connection() as conn:
with conn.cursor() as cur:
last_key = None
if last_message_id is not None:
cur.execute(
"""
SELECT created_at, id
FROM messages
WHERE chat_id = %s
AND id = %s
""",
(UUID(chat_id), UUID(last_message_id)),
)
last_key = cur.fetchone()
if last_key is None:
return None
# Разные формы SQL сохраняют индексный range scan и для generic plan.
# В SQL подставляется только константа; значения остаются параметрами.
cursor_condition = ""
params = [UUID(chat_id)]
if last_key is not None:
cursor_condition = "AND (created_at, id) > (%s, %s)"
params.extend([last_key["created_at"], last_key["id"]])
params.append(limit + 1)
cur.execute(
f"""
SELECT id, chat_id, sender_id, text, attachment_ids, created_at
FROM messages
WHERE chat_id = %s
{cursor_condition}
ORDER BY created_at, id
LIMIT %s
""",
params,
)
rows = cur.fetchall()
return [serialize_message(row) for row in rows]
+37
View File
@@ -0,0 +1,37 @@
"""Grade 4 orphan cleanup. Run periodically/manual; fails closed if DB is down.
Only originals older than one hour, no committed metadata, and no active uploader.
"""
import json
from datetime import datetime, timezone, timedelta
from .database import open_database, close_database, get_connection
from .storage import s3, BUCKET
def cleanup():
cutoff = datetime.now(timezone.utc) - timedelta(hours=1)
result = dict(scanned=0, fresh=0, referenced=0, busy=0, deleted=0)
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='originals/'):
for obj in page.get('Contents', []):
result['scanned'] += 1
if obj['LastModified'] >= cutoff:
result['fresh'] += 1
continue
key = obj['Key']
with get_connection() as conn:
with conn.transaction():
locked = conn.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0)) AS locked', (key,)).fetchone()['locked']
if not locked:
result['busy'] += 1
elif conn.execute('SELECT 1 FROM attachments WHERE object_key=%s', (key,)).fetchone():
result['referenced'] += 1
else:
s3.delete_object(Bucket=BUCKET, Key=key)
result['deleted'] += 1
return result
if __name__ == '__main__':
open_database()
try:
print(json.dumps(cleanup()))
finally:
close_database()
+14
View File
@@ -0,0 +1,14 @@
"""Periodic grade 4 orphan cleanup, plus scripts/cleanup.sh for manual runs."""
import json
import time
from .database import open_database
from .cleanup import cleanup
open_database()
while True:
try:
print(json.dumps(cleanup()), flush=True)
except Exception:
# Never log endpoint/credentials or underlying SDK exception contents.
print('Cleanup unavailable; will retry in 300 seconds', flush=True)
time.sleep(300)
+48
View File
@@ -0,0 +1,48 @@
import os
from contextlib import contextmanager
from pathlib import Path
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
def get_database_url() -> str:
secret_file = os.getenv("DATABASE_URL_FILE")
if secret_file:
return Path(secret_file).read_text().strip()
database_url = os.getenv("DATABASE_URL")
if database_url:
return database_url
raise RuntimeError("Database connection is not configured")
pool = ConnectionPool(
conninfo=get_database_url(),
min_size=1,
max_size=10,
timeout=5,
open=False,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": "-c statement_timeout=5000",
},
)
def open_database():
pool.open(wait=True, timeout=15)
def close_database():
pool.close()
@contextmanager
def get_connection():
with pool.connection() as connection:
yield connection
+24
View File
@@ -0,0 +1,24 @@
"""Isolated image decoder: bounded address space, CPU and parent wall timeout."""
import resource
import sys
import warnings
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
resource.setrlimit(resource.RLIMIT_CPU, (8, 8))
from PIL import Image
Image.MAX_IMAGE_PIXELS = 40_000_000
warnings.simplefilter('error', Image.DecompressionBombWarning)
try:
with Image.open(sys.argv[1]) as image:
if image.width * image.height > 40_000_000:
sys.exit(13)
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
if actual != sys.argv[2] or getattr(image, 'n_frames', 1) != 1:
sys.exit(15)
image.verify()
with Image.open(sys.argv[1]) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
sys.exit(13)
except Exception:
sys.exit(15)
+517
View File
@@ -0,0 +1,517 @@
"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили."""
import base64
import binascii
import json
from pathlib import Path
import logging
import re
from uuid import UUID, uuid4
from fastapi import FastAPI, HTTPException, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException as StarletteHTTPException
from psycopg import OperationalError, Error as DatabaseError
from psycopg_pool import PoolTimeout
from .auth import (
create_token,
format_datetime,
hash_password,
hash_token,
parse_basic,
parse_bearer,
session_expiration,
verify_password,
)
from .database import open_database, close_database, get_connection
from .user_repository import (
create_user as insert_user,
create_session,
find_session_user_cached,
find_user_by_username,
revoke_session,
serialize_user,
)
from .chat_repository import (
create_chat as insert_chat,
get_chat as find_chat,
list_chats as find_chats,
create_message as insert_message,
list_messages as find_messages,
)
app = FastAPI(title="MTUSI Messenger", version="1.4.0")
# Serve the mandatory contract in Swagger, including auth and request schemas.
OPENAPI_SCHEMA = json.loads(
(Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text()
)
app.openapi = lambda: OPENAPI_SCHEMA
INSTANCE_ID = str(uuid4())
logger = logging.getLogger("uvicorn.error")
ERROR_CODES = {
400: "invalid_request",
401: "unauthorized",
404: "not_found",
409: "conflict",
413: "payload_too_large",
415: "unsupported_media_type",
429: "rate_limited",
500: "internal_error",
503: "unavailable",
}
def error_response(request, status_code, code, message):
return JSONResponse(
status_code=status_code,
content={
"error": {
"code": code,
"message": message,
"request_id": request.state.request_id,
}
},
)
@app.middleware("http")
async def request_id_middleware(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
response = await call_next(request)
except Exception:
# Логируем путь, метод и request_id, но НЕ тело сообщения.
await run_in_threadpool(
logger.error,
"Unhandled error: %s %s request_id=%s",
request.method,
request.url.path,
request.state.request_id,
)
response = error_response(
request, 500, "internal_error", "Внутренняя ошибка сервера"
)
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(request: Request, exc: StarletteHTTPException):
response = error_response(
request,
exc.status_code,
ERROR_CODES.get(exc.status_code, "invalid_request"),
str(exc.detail),
)
if exc.headers:
response.headers.update(exc.headers)
return response
@app.exception_handler(DatabaseError)
@app.exception_handler(PoolTimeout)
async def database_error_handler(request: Request, exc):
return error_response(request, 503, "unavailable", "База данных временно недоступна")
@app.exception_handler(RequestValidationError)
async def validation_error_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Некорректные данные запроса")
async def read_json(request: Request):
"""415 — только для неправильного Content-Type; 400 — для неверного JSON."""
content_type = (
request.headers.get("content-type", "").split(";")[0].strip().lower()
)
if content_type != "application/json":
raise HTTPException(415, "Неподдерживаемый Content-Type")
try:
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > 65536:
raise HTTPException(413, "JSON слишком большой")
raw.extend(chunk)
data = json.loads(raw)
except (ValueError, UnicodeError):
raise HTTPException(400, "Неверный JSON") from None
# [], строка, число, true, false, null — это корректный JSON,
# но НЕ объект, обязательный по контракту.
if not isinstance(data, dict):
raise HTTPException(400, "Ожидается JSON-объект")
return data
def get_current_user(request: Request):
token = parse_bearer(request.headers.get("Authorization"))
if token is None:
raise HTTPException(
401,
"Требуется действующий Bearer-токен",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
try:
user = find_session_user_cached(hash_token(token))
except (OperationalError, PoolTimeout):
raise HTTPException(
503,
"База данных временно недоступна",
) from None
if user is None:
raise HTTPException(
401,
"Недействительная или истёкшая сессия",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def get_limit(request: Request):
"""Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500."""
limit_raw = request.query_params.get("limit", "50")
if not limit_raw.isascii() or not limit_raw.isdecimal():
raise HTTPException(400, "Неверный limit")
significant = limit_raw.lstrip("0")
if not significant or len(significant) > 3:
raise HTTPException(400, "Неверный limit")
limit = int(significant) # Теперь строка содержит максимум 3 цифры.
if not 1 <= limit <= 100:
raise HTTPException(400, "limit должен быть от 1 до 100")
return limit
def encode_cursor(first_id, last_id):
raw = f"{first_id}|{last_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def decode_cursor(cursor):
try:
if not cursor or len(cursor) > 256:
raise ValueError("empty or too long")
padded = cursor + "=" * (-len(cursor) % 4)
raw = base64.b64decode(
padded, altchars=b"-_", validate=True
).decode("ascii")
first_id, last_id = raw.split("|")
return str(UUID(first_id)), str(UUID(last_id))
except (ValueError, UnicodeError, binascii.Error):
raise HTTPException(400, "Неверный cursor") from None
# ---- Health ----
@app.get("/health/live")
def health_live():
return {"status": "ok"}
@app.get("/health/ready")
def health_ready():
try:
with get_connection() as conn:
conn.execute("SELECT 1")
from .storage import s3, BUCKET
s3.list_objects_v2(Bucket=BUCKET, Prefix="originals/", MaxKeys=1)
except (OperationalError, PoolTimeout):
raise HTTPException(503, "База данных недоступна") from None
return {"status": "ok"}
# ---- Users ----
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
data = await read_json(request)
if set(data) != {"username", "display_name", "password"}:
raise HTTPException(400, "Неверные поля пользователя")
username = data["username"]
display_name = data["display_name"]
password = data["password"]
if not isinstance(username, str) or not re.fullmatch(
r"[a-z0-9_]{3,32}", username
):
raise HTTPException(400, "Недопустимый username")
if (
not isinstance(display_name, str)
or not 1 <= len(display_name) <= 100
or not display_name.strip()
):
raise HTTPException(400, "Недопустимое имя пользователя")
if not isinstance(password, str) or not 12 <= len(password) <= 128:
raise HTTPException(400, "Пароль должен содержать 12–128 символов")
password_hash = await run_in_threadpool(hash_password, password)
user = await run_in_threadpool(insert_user, username, display_name, password_hash)
if user is None:
raise HTTPException(409, "Username уже занят")
return user
@app.get("/api/v1/users/me")
def get_me(request: Request):
return get_current_user(request)
@app.post("/api/v1/auth/sessions", status_code=201)
def login(request: Request):
credentials = parse_basic(request.headers.get("Authorization"))
if credentials is None:
raise HTTPException(
401,
"Неверные учётные данные",
headers={
"WWW-Authenticate":
'Basic realm="messenger", charset="UTF-8"'
},
)
username, password = credentials
user_row = find_user_by_username(username)
if user_row is None or not verify_password(
user_row["password_hash"], password
):
raise HTTPException(
401,
"Неверные учётные данные",
headers={
"WWW-Authenticate":
'Basic realm="messenger", charset="UTF-8"'
},
)
token = create_token()
expires_at = session_expiration()
create_session(user_row["id"], hash_token(token), expires_at)
return {
"token": token,
"token_type": "Bearer",
"expires_at": format_datetime(expires_at),
"user": serialize_user(user_row),
}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def logout(request: Request):
get_current_user(request)
token = parse_bearer(request.headers.get("Authorization"))
from .session_cache import invalidate_session
if not revoke_session(hash_token(token)):
raise HTTPException(
401,
"Недействительная сессия",
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
)
invalidate_session(hash_token(token))
from fastapi import Response
return Response(status_code=204)
# ---- Chats ----
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await run_in_threadpool(get_current_user, request)
data = await read_json(request)
if set(data) != {"title", "member_ids"}:
raise HTTPException(400, "Неверные поля чата")
title = data["title"]
member_ids = data["member_ids"]
if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip():
raise HTTPException(400, "Недопустимое название чата")
if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100:
raise HTTPException(400, "Неверный список участников")
normalized_ids = []
for member_id in member_ids:
if not isinstance(member_id, str):
raise HTTPException(400, "Неверный ID участника")
try:
normalized_ids.append(str(UUID(member_id)))
except ValueError:
raise HTTPException(400, "Неверный UUID участника") from None
if len(normalized_ids) != len(set(normalized_ids)):
raise HTTPException(400, "Участники повторяются")
if user["id"] not in normalized_ids:
raise HTTPException(400, "Создатель должен быть участником")
chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids)
if chat is None:
raise HTTPException(400, "Неизвестный участник")
return chat
@app.get("/api/v1/chats")
def list_chats(request: Request):
user = get_current_user(request)
limit = get_limit(request)
cursor = request.query_params.get("cursor")
last_chat_id = None
if cursor is not None:
cursor_user_id, last_chat_id = decode_cursor(cursor)
if cursor_user_id != user["id"]:
raise HTTPException(400, "Неверный cursor")
rows = find_chats(user["id"], limit, last_chat_id)
if rows is None:
raise HTTPException(400, "Неверный cursor")
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = (
encode_cursor(user["id"], page[-1]["id"])
if has_more and page
else None
)
return {"items": page, "next_cursor": next_cursor}
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: str, request: Request):
user = get_current_user(request)
try:
chat_id = str(UUID(chat_id))
except ValueError:
raise HTTPException(400, "Неверный UUID чата") from None
chat = find_chat(chat_id, user["id"])
if chat is None:
raise HTTPException(404, "Чат не найден")
return chat
# ---- Messages ----
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def send_message(chat_id: str, request: Request):
user = await run_in_threadpool(get_current_user, request)
chat = await run_in_threadpool(get_chat, chat_id, request)
data = await read_json(request)
if "text" not in data or set(data) - {"text", "attachment_ids"}:
raise HTTPException(400, "Неверные поля сообщения")
text = data["text"]
attachment_ids = data.get("attachment_ids", [])
if not isinstance(attachment_ids, list) or len(attachment_ids) > 10:
raise HTTPException(400, "Неверный список вложений")
try:
if any(not isinstance(a, str) for a in attachment_ids):
raise ValueError()
attachment_ids = [str(UUID(a)) for a in attachment_ids]
if len(set(attachment_ids)) != len(attachment_ids):
raise ValueError()
except ValueError:
raise HTTPException(400, "Неверные ID вложений") from None
if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids):
raise HTTPException(400, "Недопустимый текст сообщения")
key = request.headers.get("Idempotency-Key", "")
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
raise HTTPException(400, "Неверный Idempotency-Key")
message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key, attachment_ids)
if message == "conflict":
raise HTTPException(409, "Ключ использован для другого сообщения")
if message is None:
raise HTTPException(404, "Чат не найден")
return message
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(chat_id: str, request: Request):
chat = get_chat(chat_id, request)
limit = get_limit(request)
cursor = request.query_params.get("cursor")
last_message_id = None
if cursor is not None:
cursor_chat_id, last_message_id = decode_cursor(cursor)
if cursor_chat_id != chat["id"]:
raise HTTPException(400, "Неверный cursor")
rows = find_messages(chat["id"], limit, last_message_id)
if rows is None:
raise HTTPException(400, "Неверный cursor")
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = (
encode_cursor(chat["id"], page[-1]["id"])
if has_more and page
else None
)
return {"items": page, "next_cursor": next_cursor}
@app.on_event("startup")
async def start_database_pool():
await run_in_threadpool(open_database)
@app.on_event("shutdown")
async def stop_database_pool():
await run_in_threadpool(close_database)
from .attachments import register
register(app, get_current_user, get_chat)
+16
View File
@@ -0,0 +1,16 @@
"""One-shot migrations, serialized and recorded in PostgreSQL."""
from pathlib import Path
import psycopg
from .database import get_database_url
with psycopg.connect(get_database_url()) as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(310026)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())")
for path in sorted(Path("/app/db/migrations").glob("*.sql")):
if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
conn.execute(path.read_text())
conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,))
print("Applied", path.name, flush=True)
else:
print("Already applied", path.name, flush=True)
+48
View File
@@ -0,0 +1,48 @@
import json
import os
from pathlib import Path
import redis
def get_cache_url():
secret_file = os.getenv("CACHE_URL_FILE")
if secret_file:
return Path(secret_file).read_text().strip()
return os.getenv("CACHE_URL", "redis://redis:6379/0")
client = redis.Redis.from_url(
get_cache_url(),
socket_connect_timeout=1,
socket_timeout=1,
decode_responses=True,
)
def get_cached_user(token_hash):
try:
value = client.get("session:" + token_hash)
return json.loads(value) if value else None
except (redis.RedisError, ValueError):
return None
def cache_user(token_hash, user, ttl):
if ttl <= 0:
return
try:
client.setex(
"session:" + token_hash,
ttl,
json.dumps(user),
)
except redis.RedisError:
pass
def invalidate_session(token_hash):
try:
client.delete("session:" + token_hash)
except redis.RedisError:
pass
+19
View File
@@ -0,0 +1,19 @@
"""Private S3 access; no automatic PUT retries, explicit bounded timeouts."""
import os
from pathlib import Path
import boto3
from botocore.config import Config
def secret(name):
return Path(os.environ[name + '_FILE']).read_text().strip()
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
s3 = boto3.client(
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
s3={'addressing_style': 'path'}),
)
+136
View File
@@ -0,0 +1,136 @@
from uuid import uuid4
from psycopg.errors import UniqueViolation
from .database import get_connection
def serialize_user(row):
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": row["created_at"].isoformat(
timespec="microseconds"
).replace("+00:00", "Z"),
}
def create_user(username, display_name, password_hash):
with get_connection() as conn:
with conn.cursor() as cur:
try:
cur.execute(
"""
INSERT INTO users (
id, username, display_name, password_hash
)
VALUES (%s, %s, %s, %s)
RETURNING id, username, display_name, created_at
""",
(uuid4(), username, display_name, password_hash),
)
return serialize_user(cur.fetchone())
except UniqueViolation:
conn.rollback()
return None
def find_user_by_username(username):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, username, display_name,
password_hash, created_at
FROM users
WHERE username = %s
""",
(username,),
)
return cur.fetchone()
def create_session(user_id, token_hash, expires_at):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO sessions (
id, user_id, token_hash, expires_at
)
VALUES (%s, %s, %s, %s)
""",
(uuid4(), user_id, token_hash, expires_at),
)
def revoke_session(token_hash):
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE sessions
SET revoked_at = NOW()
WHERE token_hash = %s
AND revoked_at IS NULL
AND expires_at > NOW()
RETURNING id
""",
(token_hash,),
)
return cur.fetchone() is not None
def find_session_user_cached(token_hash):
from datetime import datetime, timezone
from .session_cache import cache_user, get_cached_user
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
s.user_id,
s.expires_at
FROM sessions s
WHERE s.token_hash = %s
AND s.revoked_at IS NULL
AND s.expires_at > NOW()
""",
(token_hash,),
)
session = cur.fetchone()
if session is None:
return None
cached = get_cached_user(token_hash)
if cached is not None and str(cached.get("id")) == str(session["user_id"]):
return cached
with get_connection() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, username, display_name, created_at
FROM users
WHERE id = %s
""",
(session["user_id"],),
)
row = cur.fetchone()
if row is None:
return None
user = serialize_user(row)
ttl = int(
(session["expires_at"] - datetime.now(timezone.utc)).total_seconds()
)
if ttl > 0:
cache_user(token_hash, user, ttl)
return user
+65
View File
@@ -0,0 +1,65 @@
"""Stop exactly the API that returned upload 201, download via survivor; repeat both."""
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
from faults import compose, ready
def mappings():
result = {}
for container in compose('ps', '-q', 'api').split():
identity = subprocess.check_output(['docker', 'exec', container, 'python', '-c', 'import urllib.request;print(urllib.request.urlopen("http://localhost:8080/health/live").headers["X-Instance-Id"])'], text=True).strip()
result[identity] = container
return result
def main():
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=20)
alice,bob,eve,chat = c.bootstrap()
image=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
results=[]
stopped=set()
try:
for _ in range(2):
mapping=mappings()
for attempt in range(20):
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,actor=alice,raw=image,headers={'Content-Type':'image/png'},schema='Attachment')
instance=response['headers']['X-Instance-Id']
container=mapping[instance]
if container not in stopped:
break
else:
raise RuntimeError('could not reach other uploader')
att=response['json']
env=dict(os.environ,TARGET_INSTANCE_ID=instance)
subprocess.run(['scripts/stop-one.sh'],env=env,check=True)
status=subprocess.check_output(['docker','inspect','--format','{{.State.Running}}',container],text=True).strip()
check(status=='false','uploader still running')
deadline=time.monotonic()+10
while True:
try:
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
break
except AssertionError:
if time.monotonic()>deadline:
raise
time.sleep(.2)
check(download['headers']['X-Instance-Id']!=instance,'served by stopped uploader')
check(download['bytes']==image,'original changed after uploader stop')
results.append({'uploader_instance':instance,'stopped_container':container[:12],'survivor_instance':download['headers']['X-Instance-Id'],'sha256':hashlib.sha256(download['bytes']).hexdigest(),'original_identical':True})
stopped.add(container)
compose('start','api')
time.sleep(4)
ready(c)
print(json.dumps({'passed':True,'both_uploaders_tested':len(stopped),'scenarios':results},indent=2))
finally:
compose('start','api')
if __name__=='__main__':
main()
+23
View File
@@ -0,0 +1,23 @@
"""Verify every committed original against PostgreSQL, without printing keys/tokens."""
import json
from faults import remote
code = '''from src.database import open_database,get_connection
from src.storage import s3,BUCKET
import hashlib,json
open_database()
with get_connection() as conn:
rows=conn.execute('SELECT object_key,size_bytes,sha256 FROM attachments').fetchall()
checked=total=0
for row in rows:
body=s3.get_object(Bucket=BUCKET,Key=row['object_key'])['Body']
digest=hashlib.sha256(); size=0
try:
for chunk in body.iter_chunks(65536):
size+=len(chunk); digest.update(chunk)
finally:
body.close()
assert size==row['size_bytes'] and digest.hexdigest()==row['sha256'], 'original mismatch'
checked+=1; total+=size
print(json.dumps({'verified_originals':checked,'verified_bytes':total,'all_checksums_match':True}))'''
print(json.dumps(json.loads(remote(code)),indent=2))
+209
View File
@@ -0,0 +1,209 @@
"""Real lab4-only fault injection. Restores services/triggers in finally.
Operator invokes explicitly; never removes volumes or user objects.
"""
import base64
import json
import os
from pathlib import Path
import subprocess
import sys
import time
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ['docker', 'compose', '--env-file', str(ROOT / '.env')]
def compose(*args, data=None):
result = subprocess.run(COMPOSE + list(args), input=data, text=True, capture_output=True, cwd=ROOT)
if result.returncode:
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
return result.stdout.strip()
def sql(statement):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', data=statement)
def remote(code, data=None):
return compose('exec', '-T', 'api', 'python', '-c', code, data=data)
def ready(c):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready')
return
except AssertionError:
time.sleep(.5)
raise RuntimeError('readiness timeout')
S3_SUMMARY = '''from src.storage import s3, BUCKET
import json
objects = [o['Key'] for p in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='originals/') for o in p.get('Contents', [])]
print(json.dumps(objects))'''
def main():
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=40)
alice, bob, eve, chat = c.bootstrap()
attachment, image = c.attachment(alice, chat)
passed = []
# Only our unique filename will trigger a database failure AFTER a real S3 PUT.
filename = 'fault-' + chat['id'] + '.png'
trigger = f'''CREATE OR REPLACE FUNCTION student_fail_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.filename = '{filename}' THEN RAISE EXCEPTION 'synthetic attachment failure'; END IF; RETURN NEW; END $$;
CREATE TRIGGER student_fail_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_fail_attachment();'''
try:
before = set(json.loads(remote(S3_SUMMARY)))
referenced = set(json.loads(remote('''from src.database import open_database, get_connection
open_database()
import json
with get_connection() as conn:
print(json.dumps([r['object_key'] for r in conn.execute('SELECT object_key FROM attachments').fetchall()]))''')))
sql(trigger)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, 503, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
after = set(json.loads(remote(S3_SUMMARY)))
orphan = after - before
check(len(orphan) == 1, 'did not observe a real orphan after DB rollback')
passed.append('S3_PUT_then_DB_failure_returns_503_with_real_orphan')
# Fresh objects survive a real cleanup. Clock advance is confined to this test
# process: production grace period and object timestamps are not changed.
result = json.loads(remote('''from src.database import open_database
open_database()
from src.cleanup import cleanup
import json
print(json.dumps(cleanup()))'''))
check(result['deleted'] == 0 and result['fresh'] > 0, 'fresh object deleted')
passed.append('fresh_orphan_survives_one_hour_grace')
result = json.loads(remote('''from src.database import open_database
open_database()
import src.cleanup as cleaner
from datetime import datetime, timezone, timedelta
class Future(datetime):
@classmethod
def now(cls, tz=None):
return datetime.now(timezone.utc) + timedelta(hours=2)
cleaner.datetime = Future
import json
print(json.dumps(cleaner.cleanup()))'''))
check(result['deleted'] >= 1 and result['referenced'] > 0, 'cleanup failed to preserve references / delete orphan')
final = set(json.loads(remote(S3_SUMMARY)))
check(not orphan & final and referenced <= final, 'cleanup lost used data')
passed.append('expired_orphan_deleted_referenced_objects_preserved_test_clock')
repeat = json.loads(remote('''from src.database import open_database
open_database()
from src.cleanup import cleanup
import json
print(json.dumps(cleanup()))'''))
check(repeat['deleted'] == 0, 'cleanup not repeatable')
passed.append('cleanup_repeatable')
# API credentials: anonymous GET/list denied, bucket policy/admin denied.
privacy = json.loads(remote('''from src.storage import s3, BUCKET
from botocore import UNSIGNED
from botocore.config import Config
from botocore.exceptions import ClientError
import boto3, json, urllib.request, urllib.error
key=s3.list_objects_v2(Bucket=BUCKET,Prefix='originals/',MaxKeys=1)['Contents'][0]['Key']
result=[]
for suffix in ('/'+BUCKET+'/'+key, '/'+BUCKET+'?list-type=2'):
try:
urllib.request.urlopen('http://s3:9000'+suffix,timeout=3)
result.append(200)
except urllib.error.HTTPError as e:
result.append(e.code)
try:
s3.get_bucket_policy(Bucket=BUCKET)
result.append(200)
except ClientError as e:
result.append(e.response['ResponseMetadata']['HTTPStatusCode'])
print(json.dumps(result))'''))
check(privacy == [403, 403, 403], 'bucket or API credentials overexposed')
passed.append('anonymous_GET_and_LIST_denied_API_cannot_read_policy')
# Probe API itself, not a load-balancer-generated failure.
request = json.dumps({'token': alice['token'], 'chat': chat['id'], 'image': base64.b64encode(image).decode()})
probe = '''import sys,json,base64,urllib.request,urllib.error
x=json.load(sys.stdin)
r=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/'+x['chat']+'/attachments?filename=outage.png',data=base64.b64decode(x['image']),headers={'Authorization':'Bearer '+x['token'],'Content-Type':'image/png'},method='POST')
try:
response=urllib.request.urlopen(r,timeout=20)
except urllib.error.HTTPError as e:
response=e
print(json.dumps({'status':response.code,'body':json.loads(response.read()),'request_id':bool(response.headers.get('X-Request-Id')),'instance_id':bool(response.headers.get('X-Instance-Id'))}))'''
for service in ('s3', 'postgres'):
compose('stop', service)
try:
failure = json.loads(remote(probe, data=request))
check(failure['status'] == 503 and failure['request_id'] and failure['instance_id'], service + ' outage mishandled')
check(failure['body']['error']['code'] == 'unavailable', 'wrong error format')
finally:
compose('start', service)
ready(c)
passed.append(service + '_outage_API_503')
# An actual disconnect cannot create an object because PUT follows full validation.
before = set(json.loads(remote(S3_SUMMARY)))
import socket
from urllib.parse import urlsplit
url = urlsplit(c.base)
con = socket.create_connection((url.hostname, url.port))
con.sendall((f'POST /api/v1/chats/{chat["id"]}/attachments?filename=aborted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {alice["token"]}\r\nContent-Type: image/png\r\nContent-Length: {len(image)}\r\n\r\n').encode() + image[:100])
con.close()
time.sleep(2)
check(before == set(json.loads(remote(S3_SUMMARY))), 'aborted upload left an S3 object')
passed.append('interrupted_upload_no_S3_object')
# Kill the process while the metadata INSERT sleeps after PUT.
sql(f'''CREATE OR REPLACE FUNCTION student_fail_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.filename = '{filename}' THEN PERFORM pg_sleep(20); END IF; RETURN NEW; END $$;
CREATE TRIGGER student_fail_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_fail_attachment();''')
before = set(json.loads(remote(S3_SUMMARY)))
with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(c.call, 'POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+filename, 201, actor=alice, raw=image, headers={'Content-Type':'image/png'})
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
observed = set(json.loads(remote(S3_SUMMARY))) - before
if observed:
break
time.sleep(.2)
check(len(observed) == 1, 'PUT not observed before crash')
compose('kill', '-s', 'SIGKILL', 'api')
try:
future.result()
raise RuntimeError('crashed request unexpectedly completed')
except AssertionError:
pass
compose('up', '-d', '--no-deps', 'api')
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
ready(c)
check(int(sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"';")) == 0, 'crash committed metadata')
check(observed <= set(json.loads(remote(S3_SUMMARY))), 'crash orphan not observed')
passed.append('API_SIGKILL_after_PUT_before_COMMIT_orphan_recoverable')
crash_cleanup = json.loads(remote('''from src.database import open_database
open_database()
import src.cleanup as cleaner
from datetime import datetime, timezone, timedelta
class Future(datetime):
@classmethod
def now(cls, tz=None):
return datetime.now(timezone.utc) + timedelta(hours=2)
cleaner.datetime = Future
import json
print(json.dumps(cleaner.cleanup()))'''))
check(crash_cleanup['deleted'] >= 1, 'crash orphan cleanup did not run')
check(not observed & set(json.loads(remote(S3_SUMMARY))), 'crash orphan not removed')
passed.append('API_crash_orphan_actually_cleaned_after_test_clock_grace')
got = c.call('GET', '/api/v1/attachments/'+attachment['id']+'/content', actor=bob)['bytes']
check(got == image, 'confirmed attachment lost during failures')
passed.append('confirmed_original_survives_faults')
print(json.dumps({'passed': passed, 'count': len(passed), 'privacy_status_codes': privacy, 'cleanup_expired_result': result, 'grace_test': 'test process clock +2h; production grace remains 1h'}, indent=2))
finally:
compose('start', 'postgres', 's3')
compose('up', '-d', '--no-deps', 'api')
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
if __name__ == '__main__':
main()
+3
View File
@@ -0,0 +1,3 @@
sample.jpg — синтетический RGB JPEG 32×24, однотонный, создан Pillow 11.2.1
в контейнере lab4. Без персональных данных; проверяет второй разрешённый формат.
Публичный PNG fixture не изменён.
Binary file not shown.

After

Width:  |  Height:  |  Size: 643 B

+104
View File
@@ -0,0 +1,104 @@
"""Black-box grade 4 checks; public tests stay untouched. Standard library only."""
import hashlib
import http.client
import json
import os
from pathlib import Path
import socket
import struct
import sys
import time
import uuid
import zlib
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import quote, urlsplit
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client, check
def png(width, height, pixels=None):
def chunk(kind, data):
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
raw = pixels if pixels is not None else b'\0' * ((width * 3 + 1) * height)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b'')
def main():
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=45)
alice, bob, eve, chat = c.bootstrap()
base = '/api/v1/chats/' + chat['id']
path = base + '/attachments?filename='
image = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
passed = []
def upload(name, data, status, mime='image/png'):
return c.call('POST', path + quote(name, safe=''), status, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None)
cases = [('bytes_over_10MiB', 'large.png', b'x' * (10485760 + 1), 413, 'image/png'),
('pixels_over_40M', 'bomb.png', png(40000001, 1, b'\0'), 413, 'image/png'),
('broken_png', 'broken.png', image[:70], 415, 'image/png'),
('broken_jpeg', 'broken.jpg', b'\xff\xd8\xff\xe0truncated', 415, 'image/jpeg'),
('mime_mismatch', 'wrong.jpg', image, 415, 'image/jpeg'),
('invalid_format', 'bad.png', b'plain text', 415, 'image/png'),
('multipart_rejected', 'bad.png', image, 415, 'multipart/form-data')]
for name, filename, data, status, mime in cases:
upload(filename, data, status, mime)
passed.append(name)
for filename in ('../x.png', 'x\\y.png', 'x\0.png', 'x\n.png', '\x7f.png', '', 'x' * 256):
upload(filename, image, 400)
passed.append('dangerous_filenames')
att, data = c.attachment(alice, chat)
for actor in (alice, bob):
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)['bytes']
check(hashlib.sha256(got).hexdigest() == att['sha256'] and got == data, 'checksum mismatch')
passed.append('original_checksum_and_shared_access')
jpeg = (Path(__file__).resolve().parent / 'fixtures/sample.jpg').read_bytes()
jpeg_att = upload('original.jpg', jpeg, 201, 'image/jpeg')['json']
downloaded = c.call('GET', '/api/v1/attachments/' + jpeg_att['id'] + '/content', actor=bob)
check(downloaded['bytes'] == jpeg and downloaded['headers'].get_content_type() == 'image/jpeg', 'JPEG original changed')
check(jpeg_att['sha256'] == hashlib.sha256(jpeg).hexdigest(), 'JPEG checksum mismatch')
passed.append('JPEG_success_and_checksum')
for suffix in ('', '/content'):
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401)
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
passed.append('download_auth_and_acl')
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json']
foreign = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'private', 'member_ids': [eve['id']]}, schema='Chat')['json']
foreign_att, _ = c.attachment(eve, foreign)
for ids in ([str(uuid.uuid4())], [foreign_att['id']]):
c.call('POST', base + '/messages', 404, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
passed.append('unknown_foreign_and_other_chat_attachment')
for ids in ([att['id']] * 2, [str(uuid.uuid4()) for _ in range(11)], 'invalid', [None]):
c.call('POST', base + '/messages', 400, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
passed.append('attachment_id_validation')
key = uuid.uuid4().hex
with ThreadPoolExecutor(max_workers=10) as pool:
messages = list(pool.map(lambda _: Client(c.base, timeout=30).message(alice, chat, '', key=key, attachments=[att['id']])['json'], range(10)))
check(all(x == messages[0] for x in messages), 'concurrent replay differs')
c.call('POST', base + '/messages', 409, actor=alice, body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': key})
key = uuid.uuid4().hex
first = c.message(alice, chat, 'old text', key=key)['json']
check(first == c.message(alice, chat, 'old text', key=key, attachments=[])['json'], 'default [] differs')
passed.append('attachment_idempotency_across_replicas')
# Actual chunked request without Content-Length exercises incremental byte limit.
url = urlsplit(c.base)
con = http.client.HTTPConnection(url.hostname, url.port, timeout=30)
con.request('POST', path + 'chunked.png', body=iter([b'x' * 65536] * 161), headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'}, encode_chunked=True)
response = con.getresponse()
check(response.status == 413, 'chunked limit failed')
response.read()
con.close()
passed.append('chunked_body_limit')
# Close mid-body before image validation / S3 PUT.
con = socket.create_connection((url.hostname, url.port))
headers = f'POST {path}interrupted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {alice["token"]}\r\nContent-Type: image/png\r\nContent-Length: {len(image)}\r\n\r\n'.encode()
con.sendall(headers + image[:100])
con.close()
time.sleep(1)
c.call('GET', '/health/ready', schema='Health')
passed.append('client_disconnect_server_survives')
output = {'passed': passed, 'count': len(passed), 'note': 'Fault/orphan checks are separate in faults.py'}
print(json.dumps(output, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()
+78
View File
@@ -0,0 +1,78 @@
"""Ten barrier-synchronized uploads + cgroup v2 memory/CPU sampling, lab4 only."""
import json
import math
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
sys.path.insert(0, str(Path(__file__).resolve().parent))
from negative import png
from client import Client
def main():
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=45)
alice, bob, eve, chat = c.bootstrap()
# Random RGB is poorly compressible; ~7.3 MiB per original, no fixture in Git.
width = height = 1600
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
image = png(width, height, raw)
del raw
compose = ['docker', 'compose', '--env-file', '.env']
containers = subprocess.check_output(compose + ['ps', '-q', 'api'], text=True).split()
code = '''import json,time
from pathlib import Path
root=Path('/sys/fs/cgroup')
def cpu():
return int(dict(line.split() for line in (root/'cpu.stat').read_text().splitlines())['usage_usec'])
start=time.monotonic(); first=cpu(); values=[]
while time.monotonic()-start < 20:
values.append(int((root/'memory.current').read_text()))
time.sleep(.02)
print(json.dumps({'sample_count':len(values),'sampled_baseline_bytes':values[0],'sampled_peak_bytes':max(values),'cgroup_lifetime_peak_bytes':int((root/'memory.peak').read_text()),'elapsed_seconds':time.monotonic()-start,'cpu_usage_seconds':(cpu()-first)/1e6}))'''
samplers = [subprocess.Popen(['docker', 'exec', name, 'python', '-c', code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for name in containers]
time.sleep(1)
barrier = threading.Barrier(10)
def worker(index):
local = Client(c.base, timeout=45)
barrier.wait()
started = time.monotonic()
try:
response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=load-'+str(index)+'.png', 201, actor=alice, raw=image, headers={'Content-Type':'image/png'}, schema='Attachment')
return {'ok': True, 'seconds': time.monotonic()-started, 'instance': response['headers']['X-Instance-Id']}
except Exception as exc:
return {'ok': False, 'seconds': time.monotonic()-started, 'error': type(exc).__name__}
started = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(worker, range(10)))
elapsed = time.monotonic() - started
metrics = []
for name, proc in zip(containers, samplers):
out, err = proc.communicate(timeout=30)
if proc.returncode:
raise RuntimeError('cgroup sampler failed')
record = json.loads(out)
record['container'] = name[:12]
record['average_cpu_percent_one_core'] = 100 * record['cpu_usage_seconds'] / record['elapsed_seconds']
metrics.append(record)
latencies = sorted(x['seconds'] for x in results if x['ok'])
successes = len(latencies)
report = {'utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'concurrency': 10, 'requests': 10,
'original_bytes': len(image), 'dimensions': [width,height], 'successful_requests': successes,
'failed_requests': 10-successes, 'elapsed_seconds': elapsed, 'successful_rps': successes/elapsed,
'latency_success_seconds': {p: latencies[max(0, math.ceil(q*len(latencies))-1)] if latencies else None for p,q in [('p50',.5),('p95',.95),('p99',.99)]},
'api_cgroups': metrics, 'requests_detail': results,
'limitations': 'One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.',
'docker_host': subprocess.check_output(['docker','info','--format','{{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}'],text=True).strip()}
text=json.dumps(report,indent=2)
Path('evidence/upload-load.json').write_text(text+'\n')
print(text)
if successes != 10:
raise SystemExit(1)
if __name__=='__main__':
main()