ЛР 4 — БПИ2301 — Дугина — на 4 #82
@@ -18,3 +18,6 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
DEFENSE.md
|
||||
student-tests
|
||||
|
||||
+12
-19
@@ -1,20 +1,13 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Local settings. Generate .env and secrets with: python3 scripts/init-local.py
|
||||
COMPOSE_PROJECT_NAME=mtusi2026lab4
|
||||
PUBLIC_PORT=8084
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Compose supplies these paths/settings to containers; secret values never go here.
|
||||
# DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# CACHE_URL=redis://redis:6379/0 (profile cache; sessions are authoritative in PostgreSQL)
|
||||
# S3_ENDPOINT=http://s3:9000
|
||||
# S3_REGION=us-east-1
|
||||
# S3_BUCKET=messenger-attachments
|
||||
# S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
# S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Fixed contract limits: 10485760 bytes, 40000000 pixels.
|
||||
|
||||
@@ -30,3 +30,5 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
DEFENSE.md
|
||||
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
FROM python:3.11.16-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV APP_HOST=0.0.0.0
|
||||
ENV APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
|
||||
COPY --chown=appuser:appuser src/ ./src/
|
||||
|
||||
COPY --chown=appuser:appuser db/ ./db/
|
||||
|
||||
COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
|
||||
|
||||
CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2", "--limit-concurrency", "64"]
|
||||
@@ -1,37 +1,281 @@
|
||||
# Отчёт — лабораторная 4
|
||||
# Лабораторная №4 — S3-хранилище и вложения
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
Автор: **Дугина Александра, БПИ2301**. Дисциплина: «Высоконагруженные приложения», МТУСИ.
|
||||
Целевая оценка: **4**.
|
||||
|
||||
## Запуск из чистого clone
|
||||
## Запуск
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker Engine/Compose v2, Python 3.9+ для публичных проверок и make.
|
||||
В контейнере используется Python 3.11.16. На macOS должен работать Docker Desktop.
|
||||
|
||||
## Архитектура и решения
|
||||
```sh
|
||||
python3 scripts/init-local.py
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make check
|
||||
make validate PYTHON=.venv/bin/python
|
||||
# На macOS для credential helper при необходимости:
|
||||
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
scripts/compose.sh ps
|
||||
curl -i http://localhost:8084/health/ready
|
||||
make test BASE_URL=http://localhost:8084
|
||||
make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
|
||||
scripts/compose.sh start api
|
||||
```
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
`init-local.py` создаёт локальные случайные пароли, файлы secrets с mode 0600
|
||||
и `.env`, не перезаписывая имеющуюся конфигурацию. `.env` и secrets исключены
|
||||
из Git и контекста Docker build. Пароли и токены в evidence не сохраняются.
|
||||
Оба bootstrap запускаются Compose автоматически; повторный `up` безопасен.
|
||||
Swagger с обязательным контрактом: `http://localhost:8084/docs`.
|
||||
|
||||
## Доказательства критериев
|
||||
Полный проверочный прогон: `python3 scripts/verify.py`. Он намеренно выполняет
|
||||
отказы только lab4, создаёт синтетические данные и восстанавливает сервисы.
|
||||
Отдельные команды описаны в [scripts/README.md](scripts/README.md).
|
||||
Безопасная остановка: `scripts/compose.sh stop`; возобновление:
|
||||
`scripts/compose.sh up -d`. Скрипты не удаляют volumes или пользовательские данные.
|
||||
Команд удаления volumes в инструкции и служебных скриптах нет.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
## Переход от lab3
|
||||
|
||||
Источник — собственный соседний `../mtusi2026lab3`, только чтение. Перенесены:
|
||||
`src/auth.py`, `database.py`, `user_repository.py`, `session_cache.py`,
|
||||
основа `main.py` и `chat_repository.py`, SQL-миграции 001/002, Dockerfile,
|
||||
requirements и HAProxy/Compose. Сохранены Argon2id, Basic-вход, непрозрачные
|
||||
256-битные Bearer-токены, PostgreSQL-сессии с TTL, Redis-кеш профилей,
|
||||
чаты/участники, курсорная история, транзакционная идемпотентность сообщений,
|
||||
балансировка, обязательные заголовки и healthchecks.
|
||||
|
||||
В lab4 адаптированы сообщения к `attachment_ids`, добавлены S3 и очистка.
|
||||
Старые POST `{"text":"..."}` эквивалентны запросам с `attachment_ids: []`.
|
||||
При конкурентном повторе ключа через разные реплики создаётся одно сообщение;
|
||||
сравнение повторного body включает текст и упорядоченный список вложений.
|
||||
Сообщение и запись идемпотентности фиксируются одной транзакцией.
|
||||
Срок хранения ключей не ограничен уборщиком: гарантируется минимум требуемых 24 часов.
|
||||
|
||||
Исходный lab3, его `.env`, secrets, volumes, venv и evidence не изменялись и не
|
||||
копировались как результаты lab4. Публичные `tests/`, `contracts/`, требования
|
||||
и критерии текущего skeleton не изменены. Все новые тесты — в `student-tests/`.
|
||||
|
||||
## Архитектура и версии
|
||||
|
||||
```text
|
||||
Клиент → 127.0.0.1:8084 → HAProxy → FastAPI × 2
|
||||
├─ PostgreSQL: данные, сессии, метаданные
|
||||
├─ Redis: необязательный кеш профилей
|
||||
└─ MinIO: оригинальные PNG/JPEG
|
||||
cleanup → PostgreSQL + MinIO
|
||||
```
|
||||
|
||||
Выбран MinIO: стандартный S3 API, SDK boto3, приватные bucket и policy,
|
||||
раздельные credentials, простой persistent volume. Используется проверенный
|
||||
в этом стенде **Bitnami 2025.7.23-debian-12-r5, MinIO DEVELOPMENT.2025-07-23T15-54-02Z**, закреплённый по digest.
|
||||
Первоначальные попытки скачать release-теги и образ Quay по digest завершились
|
||||
отказами реестров. Quay-образ работал из локального кеша, но не подходил для
|
||||
чистого запуска; поэтому итоговый S3 и bootstrap используют один доступный
|
||||
Bitnami-образ. Его pull по точному digest реально выполнен повторно и включён
|
||||
в итоговый verify. Перед сменой образа сохранена локальная резервная копия
|
||||
volume в игнорируемом backups/, после смены проверены все 33 прежних оригинала
|
||||
(79 300 697 bytes), все SHA-256 совпали — storage-image-switch.txt.
|
||||
Смена образа не создавала новый пустой volume и не удаляла старые данные.
|
||||
|
||||
| Роль / service | Образ / runtime | Health / запуск | Persistent state | Лимит CPU / RAM |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Балансировщик `lb` | haproxy:3.0.9-alpine | readiness API | нет | 0,5 / 128 MiB |
|
||||
| `api` × 2 | python:3.11.16-slim-bookworm, собственная сборка | PostgreSQL + LIST S3 | нет; tmpfs 16 MiB | каждый 0,75 / 768 MiB |
|
||||
| `postgres` | postgres:16.8-bookworm | pg_isready | postgres_data | 1 / 512 MiB |
|
||||
| `redis` | redis:7.4.2-alpine | ping | redis_data, AOF always | 0,25 / 128 MiB |
|
||||
| `s3` | bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 | /minio/health/live | s3_data | 1 / 768 MiB |
|
||||
| `s3-bootstrap` | bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20 | одноразовый mc | нет | одноразовая задача |
|
||||
| `migrate` | образ API | SQL под advisory lock | schema_migrations в БД | 0,5 / 256 MiB |
|
||||
| `cleanup` | образ API | цикл 300 с, restart unless-stopped | нет | 0,25 / 256 MiB |
|
||||
|
||||
Compose project — `mtusi2026lab4`; volumes получают этот prefix. Lab3 остаётся
|
||||
на 8080. Только LB публикуется и только на loopback; порты реплик, PostgreSQL,
|
||||
Redis, S3 9000 и консоли 9001 наружу не публикуются. Сеть — отдельная
|
||||
`mtusi2026lab4_default`.
|
||||
|
||||
Библиотеки: FastAPI 0.115.12, Uvicorn 0.34.2, psycopg 3.2.6,
|
||||
psycopg-pool 3.2.6, Redis 5.2.1, argon2-cffi 23.1.0,
|
||||
boto3 1.38.12, Pillow 11.2.1; фактически установлен botocore 1.38.46.
|
||||
`.env.example` содержит настройки запуска; внутренние URL и пути secrets заданы
|
||||
в Compose. Контрактные лимиты фиксированы в коде, не повышаются через env.
|
||||
Redis не источник сессий: каждый защищённый запрос проверяет сессию в PostgreSQL.
|
||||
Отказ кеша не делает API неготовым; PostgreSQL и S3 критичны для readiness.
|
||||
|
||||
## Данные и API
|
||||
|
||||
Миграция `003_attachments.sql` добавляет таблицу Attachment: UUID, chat_id,
|
||||
owner_id, filename, уникальный object_key, size_bytes, реальный content_type,
|
||||
SHA-256, ready status, created_at и updated_at. Внешние ключи связывают её с
|
||||
пользователем и чатом. Сообщение хранит упорядоченный массив UUID вложений;
|
||||
при записи в той же транзакции проверяется существование всех ID в этом чате.
|
||||
Обязательный API не удаляет вложения и не меняет участников чата.
|
||||
|
||||
Бинарные данные хранятся только в S3 `messenger-attachments/originals/<server UUID>`.
|
||||
Имя пользователя не является путём или ключом. API не возвращает object_key,
|
||||
S3 endpoint или credentials. JSON вложения содержит `variants: []`,
|
||||
`error_code: null`, `status: ready` и все поля обязательного OpenAPI.
|
||||
|
||||
Загрузка: `POST /api/v1/chats/{chat_id}/attachments?filename=photo.png`,
|
||||
raw binary, не multipart. Обязательны Bearer и участие в чате.
|
||||
Размер проверяется по Content-Length до чтения и по каждому chunk при чтении;
|
||||
chunked upload также ограничен 10 MiB. Байты поступают во временный файл tmpfs,
|
||||
одновременно вычисляется SHA-256. На реплике только одна активная загрузка;
|
||||
остальные ждут semaphore максимум 30 с без полного чтения тела.
|
||||
|
||||
Декодер работает в отдельном Linux-процессе: Pillow проверяет формат, размеры,
|
||||
PNG CRC/структуру и полное декодирование. До load проверяется 40 млн пикселей.
|
||||
У процесса RLIMIT_AS 512 MiB, CPU 8 с, wall timeout 10 с; анимированные изображения
|
||||
не принимаются. В API ограничены tmpfs и память контейнера; Uvicorn допускает
|
||||
не более 64 конкурентных соединений. На чтение upload отведено 30 с.
|
||||
Ни файл больше 10 MiB, ни декомпрессионная бомба не читаются/декодируются без лимита.
|
||||
Временный файл закрывается и удаляется; после SIGKILL tmpfs не является persistent
|
||||
хранилищем и очищается при остановке контейнера.
|
||||
|
||||
Filename длиной 1–255 символов не может быть пустым/пробельным, содержать
|
||||
slash/backslash, NUL или управляющие/служебные символы. Несоответствующий MIME,
|
||||
битое изображение — 415, превышение bytes/pixels — 413, неверные поля — 400.
|
||||
JSON-запросы тоже читаются с лимитом 64 KiB.
|
||||
|
||||
Скачивание идёт через `GET /api/v1/attachments/{id}/content` без redirect,
|
||||
потоками по 64 KiB; SDK использует connect timeout 2 с, read timeout 5 с,
|
||||
не повторяет PUT автоматически. Ответ содержит истинный MIME, Content-Length,
|
||||
Content-Disposition: attachment, nosniff и private/no-store. Оригинал не
|
||||
перекодируется: побайтовое равенство и SHA-256 проверены для PNG и JPEG.
|
||||
Участник читает metadata/content; посторонний/неизвестный ID получает 404,
|
||||
неавторизованный — 401 с WWW-Authenticate. Ошибки API сохраняют X-Request-Id,
|
||||
X-Instance-Id и контрактный JSON без секретов/SQL/stack trace.
|
||||
|
||||
Сообщение принимает максимум 10 уникальных ID. Пустой текст допустим только
|
||||
с вложением. Неизвестные, чужие и принадлежащие другому чату ID — 404.
|
||||
Любой участник может использовать вложение своего чата, включая загруженное
|
||||
другим участником. Обязательная курсорная пагинация `(created_at,id)` сохранена;
|
||||
это не snapshot истории между запросами, при конкурентных вставках действует
|
||||
обычная keyset-семантика и серверное время PostgreSQL.
|
||||
|
||||
## Транзакционные границы и восстановление
|
||||
|
||||
Общей атомарной транзакции S3/PostgreSQL нет. Порядок:
|
||||
|
||||
1. Полностью принять и проверить файл; при обрыве до этого шага PUT не выполняется.
|
||||
2. Открыть транзакцию БД и взять advisory lock по серверному object_key.
|
||||
3. Выполнить один S3 PUT с известной длиной.
|
||||
4. Записать ready-метаданные и COMMIT PostgreSQL.
|
||||
5. Только после успешного COMMIT вернуть 201.
|
||||
|
||||
Сбой DB после PUT или завершение процесса может оставить объект без строки БД.
|
||||
Немедленного безусловного DELETE нет: при неопределённом результате COMMIT
|
||||
он мог бы удалить уже подтверждённые данные. Вместо этого работает реальная
|
||||
периодическая сверка: `src/cleanup.py`, сервис `cleanup` каждые 300 секунд.
|
||||
|
||||
Очистка обходит только prefix originals/, пропускает объекты моложе **одного часа**
|
||||
по S3 LastModified. Для старого объекта берёт неблокирующий advisory lock,
|
||||
повторно проверяет отсутствие строки Attachment и только затем удаляет объект.
|
||||
Upload держит тот же lock от PUT до COMMIT. Занятый lock пропускается.
|
||||
Недоступная БД не трактуется как «строки нет»: очистка прерывается без DELETE,
|
||||
задача повторяется позже. Пагинация LIST ограничивает объём одного обходного блока.
|
||||
Объекты с метаданными, даже пока не прикреплённые к сообщению, сохраняются.
|
||||
|
||||
Повторяемый ручной запуск: `scripts/cleanup.sh`; результат — счётчики scanned,
|
||||
fresh, referenced, busy, deleted. Их можно сравнить до/после и повторить команду.
|
||||
Проверены реальный rollback после PUT и SIGKILL во время INSERT после PUT.
|
||||
В обоих случаях обнаружен фактический orphan и проверено его удаление.
|
||||
Для проверки истечения grace часы **только тестового процесса cleanup** сдвинуты
|
||||
на +2h: системные часы, S3 timestamps и production grace не менялись.
|
||||
Реальное ожидание часа не выполнялось. Свежие файлы в обычном запуске остаются,
|
||||
используемые файлы сохраняются и при сдвиге тестовых часов.
|
||||
|
||||
S3 bucket bootstrap: `mc mb --ignore-existing`, anonymous none, отдельный
|
||||
пользователь API и policy только на один bucket/prefix. Root credentials
|
||||
отсутствуют в API. Его GET/PUT/DELETE и LIST нужны для хранения и очистки;
|
||||
управление пользователями/bucket policy не разрешено. Проверка вне API
|
||||
дала 403 на anonymous GET существующего объекта и LIST bucket; обращение
|
||||
пользователя API к bucket policy также дало 403. Повторные `up/start` действительно
|
||||
повторяли bootstrap и миграции без потери данных.
|
||||
|
||||
## Реальные проверки
|
||||
|
||||
Итоговый полный прогон `scripts/verify.py` завершился с exit code 0.
|
||||
Все 16 шагов перечислены в `evidence/verification.json`.
|
||||
|
||||
| Требование / проверка | Реальная команда / воздействие | Результат | Evidence |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Структура | make check | PASS | check.txt |
|
||||
| OpenAPI | make validate PYTHON=.venv/bin/python | PASS | validate.txt |
|
||||
| Доступный закреплённый образ | docker pull bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:… | PASS | image-pull.txt |
|
||||
| Compose | docker compose --env-file .env config --quiet | PASS, exit 0 | compose-config.txt, verification.json |
|
||||
| Сборка/миграции/bootstrap | docker compose --env-file .env up -d --build | PASS | build.txt |
|
||||
| Весь публичный сценарий lab4 | make test BASE_URL=http://localhost:8084 | PASS | smoke.txt |
|
||||
| Сессии/сообщения/файл после restart всех stores | make test-persistence … ACTION_SCRIPT=scripts/restart.sh | PASS | persistence.txt, restart-before.txt, restart-after.txt |
|
||||
| Работа при отказе API | make test-failover … ACTION_SCRIPT=scripts/stop-one.sh | PASS; 116 успешных записей, 3 переходные ошибки; восстановление 2,280 с | failover.txt |
|
||||
| Обе загрузившие реплики остановлены по очереди | python3 student-tests/attachment-failover.py | PASS, 2 сценария; SHA-256 совпал | attachment-failover.txt |
|
||||
| PNG/JPEG, оригинал, доступ другого участника | python3 student-tests/negative.py | PASS | negative.txt |
|
||||
| >10 MiB с Content-Length и chunked, >40M pixels | тот же negative.py | PASS, 413 | negative.txt |
|
||||
| Битые PNG/JPEG, MIME/format/multipart | тот же negative.py | PASS, 415 | negative.txt |
|
||||
| Filename, некорректные/повторные/лишние ID | тот же negative.py | PASS, 400 | negative.txt |
|
||||
| Неизвестное/чужое/из другого чата вложение, auth | тот же negative.py | PASS, 404/401 | negative.txt |
|
||||
| 10 конкурентных идемпотентных POST, default [] | тот же negative.py | PASS | negative.txt |
|
||||
| Отказ S3 / PostgreSQL | python3 student-tests/faults.py, реальные stop/start | PASS, API JSON 503 с headers | faults.txt |
|
||||
| DB failure после S3 PUT | trigger для одного случайного filename | PASS, orphan обнаружен и очищен | faults.txt |
|
||||
| Обрыв upload | реальный TCP close до конца body | PASS, новых S3-объектов нет | faults.txt |
|
||||
| SIGKILL после PUT до COMMIT | остановлены API, затем восстановлены | PASS, orphan обнаружен и очищен | faults.txt |
|
||||
| Grace, сохранность referenced, повторная очистка | обычный и тестовый сдвинутый clock | PASS | faults.txt, cleanup.txt, cleanup-repeat.txt |
|
||||
| Приватность и ограниченная policy | unsigned GET/LIST, API GetBucketPolicy | PASS: 403 / 403 / 403 | faults.txt |
|
||||
| Все сохранённые оригиналы после проверок | python3 student-tests/audit-storage.py | PASS; 55 оригиналов, 157 371 407 bytes; каждый SHA-256 и размер | storage-audit.txt |
|
||||
| 10 одновременных upload и RAM | python3 student-tests/upload-load.py | PASS, 10/10 | upload-load.json, upload-load.txt |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Все пути evidence в таблице относительны каталогу `evidence/`.
|
||||
Negative suite — 16 групп, fault suite — 11. Failover содержит ожидаемые временные
|
||||
ошибки при отключении реплики: они учтены, не скрыты; успешный итог означает
|
||||
восстановление и сохранность подтверждённых записей, а не отсутствие сетевого отказа.
|
||||
|
||||
## Эксперименты и отказы
|
||||
В разработке были неуспешные попытки: скачивание первоначальных тегов MinIO,
|
||||
доступ sandbox к Docker/сети, а также неверная проверка cleanup в собственном
|
||||
повторном fault-тесте. Последняя сравнивала со списком всех объектов, ошибочно
|
||||
считая orphan от предыдущей аварии используемым. Исправлена именно проверка
|
||||
сохранности на сравнение с ключами, реально указанными в PostgreSQL;
|
||||
алгоритм очистки не ослаблялся. Исходный FAIL сохранён в `faults-initial.txt`.
|
||||
В финальном прогоне FAIL нет. Публичные тесты не редактировались.
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
## Эксперимент с памятью
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Запуск: `python3 student-tests/upload-load.py`, 2026-11-06 14:53:56 UTC.
|
||||
Среда: macOS 26.6.2 arm64, Docker Linux aarch64, выделены 8 CPU и
|
||||
8 218 316 800 байт RAM. API лимитированы каждый 0,75 CPU и 768 MiB.
|
||||
Сервисы прогреты предыдущими функциональными проверками; отдельной серии
|
||||
прогрева и повторов нагрузки не было.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Сгенерирован случайный RGB PNG 1600×1600, **7 684 008 bytes**. Один общий
|
||||
файл отправлен 10 клиентами после barrier одновременно через HAProxy;
|
||||
каждая реплика обрабатывает свои загрузки последовательно через semaphore.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
| Метрика | Измерено |
|
||||
| --- | --- |
|
||||
| Конкуренция / запросы | 10 / 10 |
|
||||
| Успехи / ошибки | 10 / 0 |
|
||||
| Длительность HTTP-пачки | 1,785141 с |
|
||||
| Успешные запросы/с | 5,601799 |
|
||||
| p50 / p95 / p99 успешных | 1,090735 / 1,743270 / 1,743270 с |
|
||||
| API 1 sampled baseline / peak | 76,59 / **107,57 MiB** |
|
||||
| API 2 sampled baseline / peak | 76,99 / **118,64 MiB** |
|
||||
| Cgroup lifetime memory.peak API 1 / 2 | 131,89 / 134,75 MiB |
|
||||
| CPU time API 1 / 2 за окно ~20 с | 2,439180 / 2,514936 CPU-с |
|
||||
| Средний CPU относительно одного ядра за это окно | 12,19% / 12,57% |
|
||||
|
||||
Память измерена из cgroup v2 `memory.current` внутри каждого API-контейнера
|
||||
каждые 20 мс, 907/905 samples; включает дочерний декодер, tmpfs и page cache.
|
||||
Это наблюдённый sampled peak, не гарантия точного мгновенного максимума.
|
||||
`memory.peak` относится ко всей жизни контейнера, включая прежнюю работу;
|
||||
не выдаётся за пик исключительно этой нагрузки. Пики двух реплик могут
|
||||
не совпадать по времени, складывать их в «пик стенда» нельзя.
|
||||
CPU усреднён за окно наблюдения 20 с с простоем после короткой пачки,
|
||||
поэтому не описывает CPU только активных 1,8 с.
|
||||
|
||||
Один локальный closed-loop запуск, накладные расходы клиента, малая выборка
|
||||
и совместное использование host с другими лабораторными не позволяют
|
||||
заявлять предельную пропускную способность или production SLA. Все исходные
|
||||
числа и результаты каждого запроса — в `evidence/upload-load.json`.
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
name: mtusi2026lab4
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.8-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
restart: unless-stopped
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always]
|
||||
volumes: [redis_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 20
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
migrate:
|
||||
build: .
|
||||
image: mtusi2026lab4-api:local
|
||||
command: [python, -m, src.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
restart: 'no'
|
||||
api:
|
||||
build: .
|
||||
image: mtusi2026lab4-api:local
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3-bootstrap:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
replicas: 2
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()']
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
tmpfs:
|
||||
- /tmp:size=16m,mode=1777
|
||||
stop_grace_period: 6s
|
||||
cpus: 0.75
|
||||
mem_limit: 768m
|
||||
restart: unless-stopped
|
||||
cleanup:
|
||||
image: mtusi2026lab4-api:local
|
||||
command: [python, -m, src.cleanup_loop]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 256m
|
||||
lb:
|
||||
image: haproxy:3.0.9-alpine
|
||||
ports: ['127.0.0.1:${PUBLIC_PORT:-8084}:8080']
|
||||
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready']
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
restart: unless-stopped
|
||||
s3:
|
||||
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
entrypoint: [/opt/bitnami/minio/bin/minio]
|
||||
user: '0:0'
|
||||
command: [server, /data, --console-address, ':9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
secrets: [s3_root_user, s3_root_password]
|
||||
volumes: [s3_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, curl, -f, http://localhost:9000/minio/health/live]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
restart: unless-stopped
|
||||
mem_limit: 768m
|
||||
cpus: 1.0
|
||||
s3-bootstrap:
|
||||
image: bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
entrypoint: [/bin/sh, /bootstrap.sh]
|
||||
volumes:
|
||||
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||
- ./infra/s3-policy.json:/policy.json:ro
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: 'no'
|
||||
volumes:
|
||||
s3_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
@@ -0,0 +1,51 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name VARCHAR(100) NOT NULL
|
||||
CHECK (length(btrim(display_name)) > 0),
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(100) NOT NULL
|
||||
CHECK (length(btrim(title)) > 0),
|
||||
created_by UUID NOT NULL REFERENCES users(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id),
|
||||
joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
text TEXT NOT NULL
|
||||
CHECK (length(text) <= 4000 AND length(btrim(text)) > 0),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
|
||||
ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_chat_members_user_id
|
||||
ON chat_members (user_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
|
||||
ON sessions (expires_at);
|
||||
@@ -0,0 +1,9 @@
|
||||
CREATE TABLE message_requests (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
sender_id UUID NOT NULL REFERENCES users(id),
|
||||
key VARCHAR(128) NOT NULL,
|
||||
text TEXT NOT NULL,
|
||||
message_id UUID REFERENCES messages(id),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (chat_id, sender_id, key)
|
||||
);
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE TABLE attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id),
|
||||
owner_id UUID NOT NULL REFERENCES users(id),
|
||||
filename TEXT NOT NULL,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids UUID[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages ADD CHECK (length(text) <= 4000 AND (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0));
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"passed": true,
|
||||
"both_uploaders_tested": 2,
|
||||
"scenarios": [
|
||||
{
|
||||
"uploader_instance": "406afeb6-f014-4233-87fb-3ee2e5d0f599",
|
||||
"stopped_container": "30fcbabd1e8c",
|
||||
"survivor_instance": "8c155e11-6333-44b9-8520-b1e6e1e53485",
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"original_identical": true
|
||||
},
|
||||
{
|
||||
"uploader_instance": "8c155e11-6333-44b9-8520-b1e6e1e53485",
|
||||
"stopped_container": "050e1dff86f8",
|
||||
"survivor_instance": "807def1e-c2d4-4555-896c-68b5796f7c8f",
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
|
||||
"original_identical": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
Compose now can delegate build to bake for better performances
|
||||
Just set COMPOSE_BAKE=true
|
||||
#0 building with "desktop-linux" instance using docker driver
|
||||
|
||||
#1 [migrate internal] load build definition from Dockerfile
|
||||
#1 transferring dockerfile: 954B done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [migrate internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [migrate internal] load .dockerignore
|
||||
#3 transferring context: 222B done
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [migrate internal] load build context
|
||||
#4 transferring context: 1.34kB done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [migrate 1/8] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
|
||||
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [migrate 2/8] WORKDIR /app
|
||||
#6 CACHED
|
||||
|
||||
#7 [migrate 7/8] COPY --chown=appuser:appuser db/ ./db/
|
||||
#7 CACHED
|
||||
|
||||
#8 [migrate 3/8] COPY requirements.txt .
|
||||
#8 CACHED
|
||||
|
||||
#9 [migrate 5/8] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
#9 CACHED
|
||||
|
||||
#10 [migrate 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
#10 CACHED
|
||||
|
||||
#11 [migrate 6/8] COPY --chown=appuser:appuser src/ ./src/
|
||||
#11 CACHED
|
||||
|
||||
#12 [migrate 8/8] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
#12 CACHED
|
||||
|
||||
#13 [migrate] exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:87efe3e62f69573c004c4121ff309b1ad35bdf670075e413f766061b78b3efc5 done
|
||||
#13 exporting config sha256:7be1ef706b170dc197efbeedfbd3d09501e6a5f4e8fd590df6b3e6ff14ebdb49 done
|
||||
#13 exporting attestation manifest sha256:2e280e83286ae3dab79ddf06b9735524b172bbfdde5487869259e21b361ac693 done
|
||||
#13 exporting manifest list sha256:26a6698cdea3199fe81ad434ade082fb14ad71f20b635254f0f8ed7c3cdcba5e done
|
||||
#13 naming to docker.io/library/mtusi2026lab4-api:local done
|
||||
#13 unpacking to docker.io/library/mtusi2026lab4-api:local done
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 [migrate] resolving provenance for metadata file
|
||||
#14 DONE 0.0s
|
||||
|
||||
#15 [api internal] load build definition from Dockerfile
|
||||
#15 transferring dockerfile: 954B done
|
||||
#15 DONE 0.0s
|
||||
|
||||
#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm
|
||||
#2 DONE 0.0s
|
||||
|
||||
#16 [api internal] load .dockerignore
|
||||
#16 transferring context: 222B done
|
||||
#16 DONE 0.0s
|
||||
|
||||
#17 [api internal] load build context
|
||||
#17 transferring context: 1.34kB done
|
||||
#17 DONE 0.0s
|
||||
|
||||
#5 [api 1/8] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b
|
||||
#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 0.0s done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#18 [api 7/8] COPY --chown=appuser:appuser db/ ./db/
|
||||
#18 CACHED
|
||||
|
||||
#6 [api 2/8] WORKDIR /app
|
||||
#6 CACHED
|
||||
|
||||
#19 [api 6/8] COPY --chown=appuser:appuser src/ ./src/
|
||||
#19 CACHED
|
||||
|
||||
#20 [api 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt
|
||||
#20 CACHED
|
||||
|
||||
#21 [api 5/8] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser
|
||||
#21 CACHED
|
||||
|
||||
#22 [api 3/8] COPY requirements.txt .
|
||||
#22 CACHED
|
||||
|
||||
#23 [api 8/8] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json
|
||||
#23 CACHED
|
||||
|
||||
#24 [api] exporting to image
|
||||
#24 exporting layers done
|
||||
#24 exporting manifest sha256:36e8cc71595fc04804ec665284eb6328bd087de9dded24bf5363e43219b05be3 done
|
||||
#24 exporting config sha256:90e3dbc5cad2036701f73221a02ab439270f933b193dbd812901db095860b963 done
|
||||
#24 exporting attestation manifest sha256:2364f47deb3b19f215c9813c211c0510a5c625fa838558d4fbc002e090e306e2 done
|
||||
#24 exporting manifest list sha256:bf8b09641c0bf0889e4103bc4a7ac1dd6563feeda39b19b8762788eab50cbd5a done
|
||||
#24 naming to docker.io/library/mtusi2026lab4-api:local done
|
||||
#24 unpacking to docker.io/library/mtusi2026lab4-api:local done
|
||||
#24 DONE 0.0s
|
||||
|
||||
#25 [api] resolving provenance for metadata file
|
||||
#25 DONE 0.0s
|
||||
api Built
|
||||
migrate Built
|
||||
Container mtusi2026lab4-redis-1 Running
|
||||
Container mtusi2026lab4-s3-1 Running
|
||||
Container mtusi2026lab4-postgres-1 Running
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Created
|
||||
Container mtusi2026lab4-api-2 Stopping
|
||||
Container mtusi2026lab4-api-1 Stopping
|
||||
Container mtusi2026lab4-api-1 Stopped
|
||||
Container mtusi2026lab4-api-2 Stopped
|
||||
Container mtusi2026lab4-migrate-1 Recreate
|
||||
Container mtusi2026lab4-migrate-1 Recreated
|
||||
Container mtusi2026lab4-lb-1 Stopping
|
||||
Container mtusi2026lab4-cleanup-1 Stopping
|
||||
Container mtusi2026lab4-lb-1 Stopped
|
||||
Container mtusi2026lab4-cleanup-1 Stopped
|
||||
Container mtusi2026lab4-api-1 Recreate
|
||||
Container mtusi2026lab4-lb-1 Stopping
|
||||
Container mtusi2026lab4-cleanup-1 Stopping
|
||||
Container mtusi2026lab4-cleanup-1 Stopped
|
||||
Container mtusi2026lab4-lb-1 Stopped
|
||||
Container mtusi2026lab4-api-2 Recreate
|
||||
Container mtusi2026lab4-api-1 Recreated
|
||||
Container mtusi2026lab4-api-2 Recreated
|
||||
Container mtusi2026lab4-lb-1 Created
|
||||
Container mtusi2026lab4-cleanup-1 Recreate
|
||||
Container mtusi2026lab4-cleanup-1 Recreated
|
||||
Container mtusi2026lab4-postgres-1 Waiting
|
||||
Container mtusi2026lab4-s3-1 Waiting
|
||||
Container mtusi2026lab4-s3-1 Healthy
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab4-postgres-1 Healthy
|
||||
Container mtusi2026lab4-migrate-1 Starting
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab4-migrate-1 Started
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab4-migrate-1 Waiting
|
||||
Container mtusi2026lab4-redis-1 Waiting
|
||||
Container mtusi2026lab4-redis-1 Healthy
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab4-migrate-1 Exited
|
||||
Container mtusi2026lab4-api-2 Starting
|
||||
Container mtusi2026lab4-api-2 Started
|
||||
Container mtusi2026lab4-api-1 Starting
|
||||
Container mtusi2026lab4-api-1 Started
|
||||
Container mtusi2026lab4-api-2 Waiting
|
||||
Container mtusi2026lab4-api-1 Waiting
|
||||
Container mtusi2026lab4-api-2 Waiting
|
||||
Container mtusi2026lab4-api-1 Waiting
|
||||
Container mtusi2026lab4-api-2 Healthy
|
||||
Container mtusi2026lab4-api-1 Healthy
|
||||
Container mtusi2026lab4-lb-1 Starting
|
||||
Container mtusi2026lab4-api-2 Healthy
|
||||
Container mtusi2026lab4-api-1 Healthy
|
||||
Container mtusi2026lab4-cleanup-1 Starting
|
||||
Container mtusi2026lab4-lb-1 Started
|
||||
Container mtusi2026lab4-cleanup-1 Started
|
||||
@@ -0,0 +1,77 @@
|
||||
MODIFIED
|
||||
.dockerignore
|
||||
.env.example
|
||||
.gitignore
|
||||
REPORT.md
|
||||
infra/README.md
|
||||
|
||||
ADDED (unstaged)
|
||||
Dockerfile
|
||||
compose.yaml
|
||||
db/migrations/001_initial.sql
|
||||
db/migrations/002_idempotency.sql
|
||||
db/migrations/003_attachments.sql
|
||||
evidence/attachment-failover.txt
|
||||
evidence/build.txt
|
||||
evidence/changed-files.txt
|
||||
evidence/check.txt
|
||||
evidence/cleanup-repeat.txt
|
||||
evidence/cleanup.txt
|
||||
evidence/compose-config.txt
|
||||
evidence/development-notes.txt
|
||||
evidence/failover.txt
|
||||
evidence/faults-initial.txt
|
||||
evidence/faults.txt
|
||||
evidence/final-audit.txt
|
||||
evidence/git-status.txt
|
||||
evidence/image-pull.txt
|
||||
evidence/negative.txt
|
||||
evidence/persistence.txt
|
||||
evidence/restart-after.txt
|
||||
evidence/restart-before.txt
|
||||
evidence/restore-after-failover.txt
|
||||
evidence/smoke.txt
|
||||
evidence/storage-audit.txt
|
||||
evidence/storage-image-switch.txt
|
||||
evidence/upload-load.json
|
||||
evidence/upload-load.txt
|
||||
evidence/validate.txt
|
||||
evidence/verification.json
|
||||
infra/haproxy.cfg
|
||||
infra/s3-bootstrap.sh
|
||||
infra/s3-policy.json
|
||||
requirements.txt
|
||||
scripts/README.md
|
||||
scripts/cleanup.sh
|
||||
scripts/compose.sh
|
||||
scripts/init-local.py
|
||||
scripts/restart.sh
|
||||
scripts/stop-one.sh
|
||||
scripts/verify.py
|
||||
src/__init__.py
|
||||
src/attachments.py
|
||||
src/auth.py
|
||||
src/chat_repository.py
|
||||
src/cleanup.py
|
||||
src/cleanup_loop.py
|
||||
src/database.py
|
||||
src/image_check.py
|
||||
src/main.py
|
||||
src/migrate.py
|
||||
src/session_cache.py
|
||||
src/storage.py
|
||||
src/user_repository.py
|
||||
student-tests/attachment-failover.py
|
||||
student-tests/audit-storage.py
|
||||
student-tests/faults.py
|
||||
student-tests/fixtures/README.md
|
||||
student-tests/fixtures/sample.jpg
|
||||
student-tests/negative.py
|
||||
student-tests/upload-load.py
|
||||
|
||||
LOCAL IGNORED
|
||||
DEFENSE.md
|
||||
.env
|
||||
secrets/
|
||||
backups/
|
||||
.venv/
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1 @@
|
||||
{"scanned": 55, "fresh": 55, "referenced": 0, "busy": 0, "deleted": 0}
|
||||
@@ -0,0 +1 @@
|
||||
{"scanned": 55, "fresh": 55, "referenced": 0, "busy": 0, "deleted": 0}
|
||||
@@ -0,0 +1,13 @@
|
||||
Initial environment/development failures (not final PASS):
|
||||
- pip install in sandbox: DNS/host lookup denied. Retried with network approval; installed successfully.
|
||||
- Docker Desktop credential helper absent from PATH. scripts/compose.sh adds the existing Docker Desktop bin path.
|
||||
- minio/minio release-tag pull: access denied. Quay release-tag and cached working digest pull: 401.
|
||||
- A working cached Quay image was first used for development. Final deployment changed to accessible
|
||||
bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20.
|
||||
Docker pull succeeded. Before switching, a stopped-volume tar backup was saved locally under ignored backups/.
|
||||
All 33 previously committed originals (79300697 bytes) retained matching sizes and SHA-256 after switch.
|
||||
- Own repeated cleanup test incorrectly treated all pre-existing objects (including an orphan from an earlier
|
||||
crash) as referenced. Failure retained in faults-initial.txt. Assertion corrected to compare only keys backed
|
||||
by committed PostgreSQL metadata. Cleanup algorithm not weakened; final suite rerun.
|
||||
- Sandbox Docker socket access failures were retried with the required execution permissions.
|
||||
Public tests and mandatory contracts were not edited. No Git staging/commit/push/PR actions performed.
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8084"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 3, "first_recovery_seconds": 2.28}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,10 @@
|
||||
Traceback (most recent call last):
|
||||
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/student-tests/faults.py", line 204, in <module>
|
||||
main()
|
||||
~~~~^^
|
||||
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/student-tests/faults.py", line 92, in main
|
||||
check(not orphan & final and before <= final, 'cleanup lost used data')
|
||||
~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/Users/aleksandradugina/Desktop/лабы/4к1с/внп/Projects/mtusi2026lab4/tests/client.py", line 24, in check
|
||||
raise Failure(message)
|
||||
client.Failure: cleanup lost used data
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"passed": [
|
||||
"S3_PUT_then_DB_failure_returns_503_with_real_orphan",
|
||||
"fresh_orphan_survives_one_hour_grace",
|
||||
"expired_orphan_deleted_referenced_objects_preserved_test_clock",
|
||||
"cleanup_repeatable",
|
||||
"anonymous_GET_and_LIST_denied_API_cannot_read_policy",
|
||||
"s3_outage_API_503",
|
||||
"postgres_outage_API_503",
|
||||
"interrupted_upload_no_S3_object",
|
||||
"API_SIGKILL_after_PUT_before_COMMIT_orphan_recoverable",
|
||||
"API_crash_orphan_actually_cleaned_after_test_clock_grace",
|
||||
"confirmed_original_survives_faults"
|
||||
],
|
||||
"count": 11,
|
||||
"privacy_status_codes": [
|
||||
403,
|
||||
403,
|
||||
403
|
||||
],
|
||||
"cleanup_expired_result": {
|
||||
"scanned": 41,
|
||||
"fresh": 0,
|
||||
"referenced": 40,
|
||||
"busy": 0,
|
||||
"deleted": 1
|
||||
},
|
||||
"grace_test": "test process clock +2h; production grace remains 1h"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
$ git diff --check
|
||||
exit_code=0
|
||||
|
||||
$ git diff --exit-code -- tests contracts README.md COURSE.md CONTRIBUTING.md lab.json
|
||||
exit_code=0
|
||||
|
||||
$ git diff --cached --exit-code
|
||||
exit_code=0
|
||||
|
||||
$ git check-ignore DEFENSE.md .env secrets/postgres_password backups/minio-before-image-change.tar
|
||||
exit_code=0
|
||||
DEFENSE.md
|
||||
.env
|
||||
secrets/postgres_password
|
||||
backups/minio-before-image-change.tar
|
||||
|
||||
$ git -C ../mtusi2026lab3 status -sb
|
||||
exit_code=0
|
||||
## solution/BPI2301-Dugina...origin/solution/BPI2301-Dugina
|
||||
|
||||
PASS: 16 verification steps have actual exit_code=0; Python syntax parsed; local password values absent from tracked/nonignored files.
|
||||
@@ -0,0 +1,48 @@
|
||||
## solution/BPI2301-Dugina
|
||||
M .dockerignore
|
||||
M .env.example
|
||||
M .gitignore
|
||||
M REPORT.md
|
||||
M infra/README.md
|
||||
?? Dockerfile
|
||||
?? compose.yaml
|
||||
?? db/
|
||||
?? evidence/attachment-failover.txt
|
||||
?? evidence/build.txt
|
||||
?? evidence/changed-files.txt
|
||||
?? evidence/check.txt
|
||||
?? evidence/cleanup-repeat.txt
|
||||
?? evidence/cleanup.txt
|
||||
?? evidence/compose-config.txt
|
||||
?? evidence/development-notes.txt
|
||||
?? evidence/failover.txt
|
||||
?? evidence/faults-initial.txt
|
||||
?? evidence/faults.txt
|
||||
?? evidence/final-audit.txt
|
||||
?? evidence/git-status.txt
|
||||
?? evidence/image-pull.txt
|
||||
?? evidence/negative.txt
|
||||
?? evidence/persistence.txt
|
||||
?? evidence/restart-after.txt
|
||||
?? evidence/restart-before.txt
|
||||
?? evidence/restore-after-failover.txt
|
||||
?? evidence/smoke.txt
|
||||
?? evidence/storage-audit.txt
|
||||
?? evidence/storage-image-switch.txt
|
||||
?? evidence/upload-load.json
|
||||
?? evidence/upload-load.txt
|
||||
?? evidence/validate.txt
|
||||
?? evidence/verification.json
|
||||
?? infra/haproxy.cfg
|
||||
?? infra/s3-bootstrap.sh
|
||||
?? infra/s3-policy.json
|
||||
?? requirements.txt
|
||||
?? scripts/README.md
|
||||
?? scripts/cleanup.sh
|
||||
?? scripts/compose.sh
|
||||
?? scripts/init-local.py
|
||||
?? scripts/restart.sh
|
||||
?? scripts/stop-one.sh
|
||||
?? scripts/verify.py
|
||||
?? src/
|
||||
?? student-tests/
|
||||
@@ -0,0 +1,4 @@
|
||||
docker.io/bitnamilegacy/minio@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20: Pulling from bitnamilegacy/minio
|
||||
Digest: sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
Status: Image is up to date for bitnamilegacy/minio@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
docker.io/bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"passed": [
|
||||
"bytes_over_10MiB",
|
||||
"pixels_over_40M",
|
||||
"broken_png",
|
||||
"broken_jpeg",
|
||||
"mime_mismatch",
|
||||
"invalid_format",
|
||||
"multipart_rejected",
|
||||
"dangerous_filenames",
|
||||
"original_checksum_and_shared_access",
|
||||
"JPEG_success_and_checksum",
|
||||
"download_auth_and_acl",
|
||||
"unknown_foreign_and_other_chat_attachment",
|
||||
"attachment_id_validation",
|
||||
"attachment_idempotency_across_replicas",
|
||||
"chunked_body_limit",
|
||||
"client_disconnect_server_survives"
|
||||
],
|
||||
"count": 16,
|
||||
"note": "Fault/orphan checks are separate in faults.py"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8084"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,7 @@
|
||||
/mtusi2026lab4-api-1 2026-11-06T14:52:10.90533168Z running
|
||||
/mtusi2026lab4-api-2 2026-11-06T14:52:10.803373597Z running
|
||||
/mtusi2026lab4-cleanup-1 2026-11-06T14:52:20.417700921Z running
|
||||
/mtusi2026lab4-lb-1 2026-11-06T14:52:10.491475222Z running
|
||||
/mtusi2026lab4-postgres-1 2026-11-06T14:52:10.469402597Z running
|
||||
/mtusi2026lab4-redis-1 2026-11-06T14:52:10.394762722Z running
|
||||
/mtusi2026lab4-s3-1 2026-11-06T14:52:10.447334638Z running
|
||||
@@ -0,0 +1,7 @@
|
||||
/mtusi2026lab4-api-1 2026-11-06T14:52:01.854436551Z running
|
||||
/mtusi2026lab4-api-2 2026-11-06T14:52:01.779041218Z running
|
||||
/mtusi2026lab4-cleanup-1 2026-11-06T14:52:07.48552572Z running
|
||||
/mtusi2026lab4-lb-1 2026-11-06T14:52:07.484992512Z running
|
||||
/mtusi2026lab4-postgres-1 2026-11-06T14:44:12.610996625Z running
|
||||
/mtusi2026lab4-redis-1 2026-11-06T14:43:25.264236673Z running
|
||||
/mtusi2026lab4-s3-1 2026-11-06T14:50:28.060451049Z running
|
||||
@@ -0,0 +1,16 @@
|
||||
Container mtusi2026lab4-s3-1 Waiting
|
||||
Container mtusi2026lab4-postgres-1 Waiting
|
||||
Container mtusi2026lab4-postgres-1 Healthy
|
||||
Container mtusi2026lab4-migrate-1 Starting
|
||||
Container mtusi2026lab4-s3-1 Healthy
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Starting
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Started
|
||||
Container mtusi2026lab4-migrate-1 Started
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Waiting
|
||||
Container mtusi2026lab4-migrate-1 Waiting
|
||||
Container mtusi2026lab4-redis-1 Waiting
|
||||
Container mtusi2026lab4-s3-bootstrap-1 Exited
|
||||
Container mtusi2026lab4-redis-1 Healthy
|
||||
Container mtusi2026lab4-migrate-1 Exited
|
||||
Container mtusi2026lab4-api-2 Starting
|
||||
Container mtusi2026lab4-api-2 Started
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8084"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"verified_originals": 55,
|
||||
"verified_bytes": 157371407,
|
||||
"all_checksums_match": true
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"verified_originals": 33,
|
||||
"verified_bytes": 79300697,
|
||||
"all_checksums_match": true
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"utc": "2026-11-06T14:53:56Z",
|
||||
"concurrency": 10,
|
||||
"requests": 10,
|
||||
"original_bytes": 7684008,
|
||||
"dimensions": [
|
||||
1600,
|
||||
1600
|
||||
],
|
||||
"successful_requests": 10,
|
||||
"failed_requests": 0,
|
||||
"elapsed_seconds": 1.785140791995218,
|
||||
"successful_rps": 5.601799054080877,
|
||||
"latency_success_seconds": {
|
||||
"p50": 1.0907350000052247,
|
||||
"p95": 1.743270292005036,
|
||||
"p99": 1.743270292005036
|
||||
},
|
||||
"api_cgroups": [
|
||||
{
|
||||
"sample_count": 907,
|
||||
"sampled_baseline_bytes": 80306176,
|
||||
"sampled_peak_bytes": 112791552,
|
||||
"cgroup_lifetime_peak_bytes": 138301440,
|
||||
"elapsed_seconds": 20.002547300000515,
|
||||
"cpu_usage_seconds": 2.43918,
|
||||
"container": "050e1dff86f8",
|
||||
"average_cpu_percent_one_core": 12.194346867010967
|
||||
},
|
||||
{
|
||||
"sample_count": 905,
|
||||
"sampled_baseline_bytes": 80728064,
|
||||
"sampled_peak_bytes": 124407808,
|
||||
"cgroup_lifetime_peak_bytes": 141295616,
|
||||
"elapsed_seconds": 20.01476109199939,
|
||||
"cpu_usage_seconds": 2.514936,
|
||||
"container": "30fcbabd1e8c",
|
||||
"average_cpu_percent_one_core": 12.565406044268544
|
||||
}
|
||||
],
|
||||
"requests_detail": [
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.8693261250155047,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.4904946660099085,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.4210281249834225,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.266302750009345,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.7139787090127356,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.743270292005036,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.6961546249804087,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.9460117089911364,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.0907350000052247,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.2004604999965522,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
}
|
||||
],
|
||||
"limitations": "One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.",
|
||||
"docker_host": "aarch64 CPUs=8 RAM=8218316800"
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
{
|
||||
"utc": "2026-11-06T14:53:56Z",
|
||||
"concurrency": 10,
|
||||
"requests": 10,
|
||||
"original_bytes": 7684008,
|
||||
"dimensions": [
|
||||
1600,
|
||||
1600
|
||||
],
|
||||
"successful_requests": 10,
|
||||
"failed_requests": 0,
|
||||
"elapsed_seconds": 1.785140791995218,
|
||||
"successful_rps": 5.601799054080877,
|
||||
"latency_success_seconds": {
|
||||
"p50": 1.0907350000052247,
|
||||
"p95": 1.743270292005036,
|
||||
"p99": 1.743270292005036
|
||||
},
|
||||
"api_cgroups": [
|
||||
{
|
||||
"sample_count": 907,
|
||||
"sampled_baseline_bytes": 80306176,
|
||||
"sampled_peak_bytes": 112791552,
|
||||
"cgroup_lifetime_peak_bytes": 138301440,
|
||||
"elapsed_seconds": 20.002547300000515,
|
||||
"cpu_usage_seconds": 2.43918,
|
||||
"container": "050e1dff86f8",
|
||||
"average_cpu_percent_one_core": 12.194346867010967
|
||||
},
|
||||
{
|
||||
"sample_count": 905,
|
||||
"sampled_baseline_bytes": 80728064,
|
||||
"sampled_peak_bytes": 124407808,
|
||||
"cgroup_lifetime_peak_bytes": 141295616,
|
||||
"elapsed_seconds": 20.01476109199939,
|
||||
"cpu_usage_seconds": 2.514936,
|
||||
"container": "30fcbabd1e8c",
|
||||
"average_cpu_percent_one_core": 12.565406044268544
|
||||
}
|
||||
],
|
||||
"requests_detail": [
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.8693261250155047,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.4904946660099085,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.4210281249834225,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.266302750009345,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.7139787090127356,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.743270292005036,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.6961546249804087,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 0.9460117089911364,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.0907350000052247,
|
||||
"instance": "03ea9548-3579-43c3-83fa-8fa742019ed7"
|
||||
},
|
||||
{
|
||||
"ok": true,
|
||||
"seconds": 1.2004604999965522,
|
||||
"instance": "807def1e-c2d4-4555-896c-68b5796f7c8f"
|
||||
}
|
||||
],
|
||||
"limitations": "One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.",
|
||||
"docker_host": "aarch64 CPUs=8 RAM=8218316800"
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,164 @@
|
||||
[
|
||||
{
|
||||
"check": "check",
|
||||
"command": [
|
||||
"make",
|
||||
"check"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 0.074
|
||||
},
|
||||
{
|
||||
"check": "validate",
|
||||
"command": [
|
||||
"make",
|
||||
"validate",
|
||||
"PYTHON=.venv/bin/python"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 0.546
|
||||
},
|
||||
{
|
||||
"check": "image-pull",
|
||||
"command": [
|
||||
"docker",
|
||||
"pull",
|
||||
"bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 17.858
|
||||
},
|
||||
{
|
||||
"check": "compose-config",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"config",
|
||||
"--quiet"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 0.092
|
||||
},
|
||||
{
|
||||
"check": "build",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"up",
|
||||
"-d",
|
||||
"--build"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 19.796
|
||||
},
|
||||
{
|
||||
"check": "smoke",
|
||||
"command": [
|
||||
"make",
|
||||
"test",
|
||||
"BASE_URL=http://localhost:8084"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 1.332
|
||||
},
|
||||
{
|
||||
"check": "persistence",
|
||||
"command": [
|
||||
"make",
|
||||
"test-persistence",
|
||||
"BASE_URL=http://localhost:8084",
|
||||
"ACTION_SCRIPT=scripts/restart.sh"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 11.799
|
||||
},
|
||||
{
|
||||
"check": "failover",
|
||||
"command": [
|
||||
"make",
|
||||
"test-failover",
|
||||
"BASE_URL=http://localhost:8084",
|
||||
"ACTION_SCRIPT=scripts/stop-one.sh"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 16.266
|
||||
},
|
||||
{
|
||||
"check": "restore-after-failover",
|
||||
"command": [
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"start",
|
||||
"api"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 1.842
|
||||
},
|
||||
{
|
||||
"check": "negative",
|
||||
"command": [
|
||||
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
||||
"student-tests/negative.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 3.637
|
||||
},
|
||||
{
|
||||
"check": "faults",
|
||||
"command": [
|
||||
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
||||
"student-tests/faults.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 24.98
|
||||
},
|
||||
{
|
||||
"check": "attachment-failover",
|
||||
"command": [
|
||||
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
||||
"student-tests/attachment-failover.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 19.642
|
||||
},
|
||||
{
|
||||
"check": "upload-load",
|
||||
"command": [
|
||||
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
||||
"student-tests/upload-load.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 21.236
|
||||
},
|
||||
{
|
||||
"check": "cleanup",
|
||||
"command": [
|
||||
"scripts/cleanup.sh"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 1.036
|
||||
},
|
||||
{
|
||||
"check": "cleanup-repeat",
|
||||
"command": [
|
||||
"scripts/cleanup.sh"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 0.838
|
||||
},
|
||||
{
|
||||
"check": "storage-audit",
|
||||
"command": [
|
||||
"/opt/homebrew/opt/python@3.13/bin/python3.13",
|
||||
"student-tests/audit-storage.py"
|
||||
],
|
||||
"exit_code": 0,
|
||||
"seconds": 1.368
|
||||
}
|
||||
]
|
||||
+32
-4
@@ -1,7 +1,35 @@
|
||||
# Инфраструктура студента
|
||||
# Стенд lab4
|
||||
|
||||
Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
|
||||
Compose project `mtusi2026lab4`, внешний вход `127.0.0.1:8084`.
|
||||
Все остальные сервисы доступны только в Docker-сети этого проекта.
|
||||
API-порты, S3 и консоль MinIO 9001 не публикуются.
|
||||
Lab3 на 8080 и его volumes не используются.
|
||||
|
||||
В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
|
||||
- `lb`: HAProxy, round-robin, проверки `/health/ready`, без автоматического replay POST.
|
||||
- `api`: две реплики FastAPI без persistent volume; `/tmp` — tmpfs 16 MiB.
|
||||
- `postgres`: пользователи, сессии, чаты, сообщения, идемпотентность, Attachment.
|
||||
- `redis`: только кеш профилей; проверка сессии всегда идёт в PostgreSQL.
|
||||
- `s3`: MinIO, оригиналы в отдельном persistent volume `s3_data`.
|
||||
- `migrate`: последовательные SQL-миграции под advisory lock, таблица schema_migrations.
|
||||
- `s3-bootstrap`: mc из закреплённого Bitnami-образа, идемпотентный bucket,
|
||||
запрет anonymous доступа, отдельный пользователь API и prefix-scoped policy.
|
||||
- `cleanup`: каждые 300 секунд удаляет только orphan старше часа;
|
||||
повтор вручную: `scripts/cleanup.sh`.
|
||||
|
||||
Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
|
||||
`python3 scripts/init-local.py` генерирует локальные secrets и не перезаписывает
|
||||
существующие значения. Реальные значения исключены из Git и Docker build context.
|
||||
API не получает root credentials MinIO. `s3-policy.json` разрешает только
|
||||
PUT/GET/DELETE `originals/*` и LIST этого prefix одного bucket. DELETE и LIST
|
||||
используются механизмом очистки; создание bucket/управление policy запрещены API.
|
||||
Root credentials есть только у S3 и одноразового bootstrap.
|
||||
|
||||
Health: PostgreSQL pg_isready, Redis ping, MinIO /minio/health/live,
|
||||
API — запрос к PostgreSQL + минимальный LIST S3; LB — API readiness.
|
||||
Cleanup — периодическая CLI-задача с рестартом при завершении процесса,
|
||||
её результат виден в `docker compose logs cleanup`.
|
||||
При потере S3 обе API перестают быть ready: LB может вернуть собственный 503.
|
||||
Прямое обращение к API в fault-тесте отдельно подтверждает правильный JSON 503.
|
||||
|
||||
Безопасная остановка: `scripts/compose.sh stop`.
|
||||
Возобновление: `scripts/compose.sh up -d`.
|
||||
Volumes не удаляются ни одним служебным скриптом.
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 1024
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 60s
|
||||
timeout server 60s
|
||||
timeout check 1s
|
||||
# Never replay a potentially committed POST automatically.
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend replicas
|
||||
|
||||
backend replicas
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 1-10 api:8080 check
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy create local attachments-api /policy.json
|
||||
mc admin policy attach local attachments-api --user "$(cat /run/secrets/s3_access_key_id)"
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments/originals/*"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
],
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": [
|
||||
"originals/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.2
|
||||
pydantic==2.11.4
|
||||
psycopg[binary]==3.2.6
|
||||
psycopg-pool==3.2.6
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.38.12
|
||||
Pillow==11.2.1
|
||||
@@ -0,0 +1,41 @@
|
||||
# Команды оператора
|
||||
|
||||
Из корня lab4:
|
||||
|
||||
```sh
|
||||
python3 scripts/init-local.py
|
||||
scripts/compose.sh up -d --build
|
||||
scripts/compose.sh ps
|
||||
BASE_URL=http://localhost:8084 make test
|
||||
make test-persistence BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover BASE_URL=http://localhost:8084 ACTION_SCRIPT=scripts/stop-one.sh
|
||||
scripts/compose.sh start api
|
||||
scripts/cleanup.sh
|
||||
```
|
||||
|
||||
`compose.sh` добавляет Docker Desktop credential helper в PATH на macOS.
|
||||
`restart.sh` перезапускает все хранилища/API/LB/cleanup без удаления volumes;
|
||||
сохраняет времена запуска до/после в evidence/restart-{before,after}.txt.
|
||||
`stop-one.sh` сопоставляет TARGET_INSTANCE_ID ответу локального API внутри
|
||||
контейнера и останавливает только найденную реплику. Возврат 1 означает,
|
||||
что реплика не найдена; восстановление отдельно: `scripts/compose.sh start api`.
|
||||
|
||||
Полный проверочный прогон: `python3 scripts/verify.py`. Он намеренно выполняет
|
||||
отказы и перезапуски только проекта lab4, создаёт синтетические данные,
|
||||
сохраняет реальные exit codes/выводы в evidence. Пользовательские данные не удаляет.
|
||||
|
||||
Отдельные проверки (BASE_URL по умолчанию localhost:8084):
|
||||
|
||||
```sh
|
||||
python3 student-tests/negative.py
|
||||
python3 student-tests/faults.py
|
||||
python3 student-tests/attachment-failover.py
|
||||
python3 student-tests/upload-load.py
|
||||
```
|
||||
|
||||
`faults.py` временно создаёт DB trigger для единственного случайного имени файла,
|
||||
останавливает S3/БД, завершает API после PUT, проверяет очистку и восстанавливает
|
||||
сервисы в finally. Проверка истечения grace использует часы тестового процесса +2h:
|
||||
не меняет системное время, production-код grace или timestamps объектов.
|
||||
Не запускайте fault/нагрузочные проверки одновременно друг с другом.
|
||||
Проверки предназначены для отдельного локального учебного стенда.
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
./scripts/compose.sh exec -T api python -m src.cleanup
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then
|
||||
export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH"
|
||||
fi
|
||||
exec docker compose --env-file .env "$@"
|
||||
@@ -0,0 +1,21 @@
|
||||
"""Generate local credentials once; never overwrite an existing secret."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
directory = root / 'secrets'
|
||||
directory.mkdir(mode=0o700, exist_ok=True)
|
||||
values = {'postgres_password': secrets.token_hex(24), 's3_root_user': 'lab4root', 's3_root_password': secrets.token_hex(24), 's3_access_key_id': 'lab4api', 's3_secret_access_key': secrets.token_hex(24)}
|
||||
for name, value in values.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + '\n')
|
||||
path.chmod(0o600)
|
||||
path = directory / 'database_url'
|
||||
if not path.exists():
|
||||
password = (directory / 'postgres_password').read_text().strip()
|
||||
path.write_text(f'postgresql://messenger:{password}@postgres:5432/messenger\n')
|
||||
path.chmod(0o600)
|
||||
path = root / '.env'
|
||||
if not path.exists():
|
||||
path.write_text('COMPOSE_PROJECT_NAME=mtusi2026lab4\nPUBLIC_PORT=8084\n')
|
||||
print('Local configuration ready (secret values suppressed)')
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
containers=$(./scripts/compose.sh ps -q postgres redis s3 api lb cleanup)
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-before.txt
|
||||
./scripts/compose.sh restart postgres redis s3 api lb cleanup
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' $containers > evidence/restart-after.txt
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}"
|
||||
for container in $(./scripts/compose.sh ps -q api); do
|
||||
instance=$(docker exec "$container" python -c 'import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])')
|
||||
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop -t 2 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
exit 1
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Explicit full lab4 validation; includes lab4-only restarts and fault injection."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
os.chdir(root)
|
||||
env = dict(os.environ, BASE_URL='http://localhost:8084')
|
||||
env['PATH'] = '/Applications/Docker.app/Contents/Resources/bin:' + env.get('PATH', '')
|
||||
compose = ['docker', 'compose', '--env-file', '.env']
|
||||
checks = [
|
||||
('check', ['make', 'check']),
|
||||
('validate', ['make', 'validate', 'PYTHON=.venv/bin/python']),
|
||||
('image-pull', ['docker', 'pull', 'bitnamilegacy/minio:2025.7.23-debian-12-r5@sha256:6dabb4a2088c9a79908de3bc05f4586c23ad2182c8908e7e3acbf61c1467fb20']),
|
||||
('compose-config', compose + ['config', '--quiet']),
|
||||
('build', compose + ['up', '-d', '--build']),
|
||||
('smoke', ['make', 'test', 'BASE_URL=http://localhost:8084']),
|
||||
('persistence', ['make', 'test-persistence', 'BASE_URL=http://localhost:8084', 'ACTION_SCRIPT=scripts/restart.sh']),
|
||||
('failover', ['make', 'test-failover', 'BASE_URL=http://localhost:8084', 'ACTION_SCRIPT=scripts/stop-one.sh']),
|
||||
('restore-after-failover', compose + ['start', 'api']),
|
||||
('negative', [sys.executable, 'student-tests/negative.py']),
|
||||
('faults', [sys.executable, 'student-tests/faults.py']),
|
||||
('attachment-failover', [sys.executable, 'student-tests/attachment-failover.py']),
|
||||
('upload-load', [sys.executable, 'student-tests/upload-load.py']),
|
||||
('cleanup', ['scripts/cleanup.sh']),
|
||||
('cleanup-repeat', ['scripts/cleanup.sh']),
|
||||
('storage-audit', [sys.executable, 'student-tests/audit-storage.py']),
|
||||
]
|
||||
results = []
|
||||
try:
|
||||
for name, command in checks:
|
||||
print('RUN:', name, flush=True)
|
||||
start = time.monotonic()
|
||||
with (root / 'evidence' / (name + '.txt')).open('w') as output:
|
||||
result = subprocess.run(command, env=env, stdout=output, stderr=subprocess.STDOUT)
|
||||
record = {'check': name, 'command': command, 'exit_code': result.returncode, 'seconds': round(time.monotonic()-start, 3)}
|
||||
results.append(record)
|
||||
(root / 'evidence/verification.json').write_text(json.dumps(results, indent=2) + '\n')
|
||||
print(('PASS:' if result.returncode == 0 else 'FAIL:'), name, 'seconds=', record['seconds'], flush=True)
|
||||
if result.returncode:
|
||||
print((root / 'evidence' / (name + '.txt')).read_text(), flush=True)
|
||||
raise SystemExit(result.returncode)
|
||||
if name in ('restore-after-failover', 'attachment-failover'):
|
||||
time.sleep(4)
|
||||
finally:
|
||||
subprocess.run(compose + ['start', 'postgres', 's3', 'api'], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
@@ -0,0 +1,137 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from fastapi import HTTPException, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from .database import get_connection
|
||||
from .storage import s3, BUCKET
|
||||
|
||||
MAX_BYTES = 10 * 1024**2
|
||||
upload_slots = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
def serialize(row):
|
||||
result = {key: str(row[key]) for key in ('id', 'chat_id', 'owner_id')}
|
||||
result.update({key: row[key] for key in ('filename', 'content_type', 'size_bytes', 'sha256', 'status')})
|
||||
result.update({key: row[key].isoformat().replace('+00:00', 'Z') for key in ('created_at', 'updated_at')})
|
||||
return dict(result, variants=[], error_code=None)
|
||||
|
||||
|
||||
def persist(path, chat, owner, filename, mime, size, checksum):
|
||||
attachment_id = uuid4()
|
||||
key = 'originals/' + str(attachment_id)
|
||||
# The same lock is taken by cleanup before its final DB check/deletion.
|
||||
# A process crash releases the transaction; an uncommitted object is later cleaned.
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (key,))
|
||||
with open(path, 'rb') as source:
|
||||
s3.put_object(Bucket=BUCKET, Key=key, Body=source, ContentLength=size, ContentType=mime)
|
||||
row = conn.execute('''INSERT INTO attachments
|
||||
(id, chat_id, owner_id, filename, object_key, size_bytes, content_type, sha256)
|
||||
VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *''',
|
||||
(attachment_id, UUID(chat), UUID(owner), filename, key, size, mime, checksum)).fetchone()
|
||||
# Exiting transaction above commits before returning 201.
|
||||
return serialize(row)
|
||||
|
||||
|
||||
def accessible(attachment_id, user_id):
|
||||
try:
|
||||
aid = UUID(attachment_id)
|
||||
except ValueError:
|
||||
raise HTTPException(400, 'Неверный UUID') from None
|
||||
with get_connection() as conn:
|
||||
row = conn.execute('''SELECT a.* FROM attachments a
|
||||
JOIN chat_members cm ON cm.chat_id=a.chat_id
|
||||
WHERE a.id=%s AND cm.user_id=%s''', (aid, UUID(user_id))).fetchone()
|
||||
if row is None:
|
||||
raise HTTPException(404, 'Вложение не найдено')
|
||||
return row
|
||||
|
||||
|
||||
def register(app, current_user, get_chat):
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
|
||||
async def upload(chat_id: str, request: Request):
|
||||
user = await run_in_threadpool(current_user, request)
|
||||
chat = await run_in_threadpool(get_chat, chat_id, request)
|
||||
filename = request.query_params.get('filename', '')
|
||||
if (not 1 <= len(filename) <= 255 or not filename.strip()
|
||||
or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename)):
|
||||
raise HTTPException(400, 'Недопустимое имя файла')
|
||||
mime = request.headers.get('content-type', '').split(';')[0].strip().lower()
|
||||
if mime not in ('image/png', 'image/jpeg'):
|
||||
raise HTTPException(415, 'Ожидается PNG или JPEG')
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
if not length.isascii() or not length.isdecimal() or len(length) > 12:
|
||||
raise HTTPException(400, 'Неверный размер')
|
||||
if int(length) > MAX_BYTES:
|
||||
raise HTTPException(413, 'Файл больше 10 MiB')
|
||||
try:
|
||||
await asyncio.wait_for(upload_slots.acquire(), timeout=30)
|
||||
except asyncio.TimeoutError:
|
||||
raise HTTPException(503, 'Загрузчик занят') from None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(prefix='lab4-') as target:
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
try:
|
||||
async with asyncio.timeout(30):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise HTTPException(413, 'Файл больше 10 MiB')
|
||||
target.write(chunk)
|
||||
digest.update(chunk)
|
||||
except ClientDisconnect:
|
||||
raise HTTPException(400, 'Загрузка прервана') from None
|
||||
except TimeoutError:
|
||||
raise HTTPException(400, 'Истекло время загрузки') from None
|
||||
target.flush()
|
||||
def validate():
|
||||
try:
|
||||
return subprocess.run([sys.executable, '-m', 'src.image_check', target.name, mime],
|
||||
timeout=10, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
return 15
|
||||
code = await run_in_threadpool(validate)
|
||||
if code != 0:
|
||||
raise HTTPException(413 if code == 13 else 415, 'Недопустимое изображение')
|
||||
return await run_in_threadpool(persist, target.name, chat['id'], user['id'], filename, mime, size, digest.hexdigest())
|
||||
finally:
|
||||
upload_slots.release()
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def metadata(attachment_id: str, request: Request):
|
||||
return serialize(accessible(attachment_id, current_user(request)['id']))
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def content(attachment_id: str, request: Request):
|
||||
row = accessible(attachment_id, current_user(request)['id'])
|
||||
if request.query_params.get('variant', 'original') != 'original':
|
||||
raise HTTPException(404, 'Вариант не найден')
|
||||
obj = s3.get_object(Bucket=BUCKET, Key=row['object_key'])
|
||||
def chunks():
|
||||
try:
|
||||
for chunk in obj['Body'].iter_chunks(64 * 1024):
|
||||
yield chunk
|
||||
finally:
|
||||
obj['Body'].close()
|
||||
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
|
||||
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
|
||||
'Content-Disposition': 'attachment', 'Cache-Control': 'private, no-store'})
|
||||
|
||||
@app.exception_handler(BotoCoreError)
|
||||
@app.exception_handler(ClientError)
|
||||
async def storage_error(request, exc):
|
||||
from .main import error_response
|
||||
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
|
||||
password_hasher = PasswordHasher(
|
||||
time_cost=3,
|
||||
memory_cost=65536,
|
||||
parallelism=4,
|
||||
)
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return password_hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return password_hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def parse_basic(authorization: str | None) -> tuple[str, str] | None:
|
||||
if not authorization:
|
||||
return None
|
||||
|
||||
scheme, separator, credentials = authorization.partition(" ")
|
||||
|
||||
if not separator or scheme.lower() != "basic":
|
||||
return None
|
||||
|
||||
try:
|
||||
raw = base64.b64decode(credentials, validate=True)
|
||||
decoded = raw.decode("utf-8")
|
||||
username, password = decoded.split(":", 1)
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
return None
|
||||
|
||||
if not re.fullmatch(r"[a-z0-9_]{3,32}", username):
|
||||
return None
|
||||
|
||||
if not 12 <= len(password) <= 128:
|
||||
return None
|
||||
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(authorization: str | None) -> str | None:
|
||||
if not authorization:
|
||||
return None
|
||||
|
||||
scheme, separator, token = authorization.partition(" ")
|
||||
|
||||
if not separator or scheme.lower() != "bearer":
|
||||
return None
|
||||
|
||||
if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512:
|
||||
return None
|
||||
|
||||
return token
|
||||
|
||||
|
||||
def create_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("ascii")).hexdigest()
|
||||
|
||||
|
||||
def session_expiration() -> datetime:
|
||||
ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
if ttl < 1:
|
||||
raise ValueError("SESSION_TTL_SECONDS must be positive")
|
||||
|
||||
return datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
|
||||
|
||||
def format_datetime(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z")
|
||||
@@ -0,0 +1,245 @@
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
|
||||
def serialize_chat(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(member_id) for member_id in row["member_ids"]],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def serialize_message(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"attachment_ids": [str(a) for a in row["attachment_ids"]],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def create_chat(creator_id, title, member_ids):
|
||||
chat_id = uuid4()
|
||||
members = [UUID(member_id) for member_id in member_ids]
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM users
|
||||
WHERE id = ANY(%s::uuid[])
|
||||
""",
|
||||
(members,),
|
||||
)
|
||||
existing = {row["id"] for row in cur.fetchall()}
|
||||
|
||||
if len(existing) != len(members):
|
||||
return None
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO chats (id, title, created_by)
|
||||
VALUES (%s, %s, %s)
|
||||
RETURNING id, title, created_at
|
||||
""",
|
||||
(chat_id, title, UUID(creator_id)),
|
||||
)
|
||||
chat = cur.fetchone()
|
||||
|
||||
cur.executemany(
|
||||
"""
|
||||
INSERT INTO chat_members (chat_id, user_id)
|
||||
VALUES (%s, %s)
|
||||
""",
|
||||
[(chat_id, member_id) for member_id in members],
|
||||
)
|
||||
|
||||
return {
|
||||
"id": str(chat["id"]),
|
||||
"title": chat["title"],
|
||||
"member_ids": member_ids,
|
||||
"created_at": chat["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def get_chat(chat_id, user_id):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
c.id,
|
||||
c.title,
|
||||
c.created_at,
|
||||
ARRAY(
|
||||
SELECT cm2.user_id
|
||||
FROM chat_members cm2
|
||||
WHERE cm2.chat_id = c.id
|
||||
ORDER BY cm2.user_id
|
||||
) AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE c.id = %s
|
||||
AND cm.user_id = %s
|
||||
""",
|
||||
(UUID(chat_id), UUID(user_id)),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return serialize_chat(row) if row else None
|
||||
|
||||
|
||||
def list_chats(user_id, limit, last_chat_id=None):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
last_key = None
|
||||
|
||||
if last_chat_id is not None:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT c.created_at, c.id
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE c.id = %s
|
||||
AND cm.user_id = %s
|
||||
""",
|
||||
(UUID(last_chat_id), UUID(user_id)),
|
||||
)
|
||||
last_key = cur.fetchone()
|
||||
|
||||
if last_key is None:
|
||||
return None
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
c.id,
|
||||
c.title,
|
||||
c.created_at,
|
||||
ARRAY(
|
||||
SELECT cm2.user_id
|
||||
FROM chat_members cm2
|
||||
WHERE cm2.chat_id = c.id
|
||||
ORDER BY cm2.user_id
|
||||
) AS member_ids
|
||||
FROM chats c
|
||||
JOIN chat_members cm
|
||||
ON cm.chat_id = c.id
|
||||
WHERE cm.user_id = %s
|
||||
AND (
|
||||
%s::timestamptz IS NULL
|
||||
OR (c.created_at, c.id) >
|
||||
(%s::timestamptz, %s::uuid)
|
||||
)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s
|
||||
""",
|
||||
(
|
||||
UUID(user_id),
|
||||
last_key["created_at"] if last_key else None,
|
||||
last_key["created_at"] if last_key else None,
|
||||
last_key["id"] if last_key else None,
|
||||
limit + 1,
|
||||
),
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [serialize_chat(row) for row in rows]
|
||||
|
||||
|
||||
def create_message(chat_id, sender_id, text, key, attachment_ids):
|
||||
# INSERT ON CONFLICT waits for a concurrent transaction on the same key.
|
||||
# READ COMMITTED's next statement sees its committed result. Both the
|
||||
# message and deduplication record commit together, or both roll back.
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
attachments = [UUID(a) for a in attachment_ids]
|
||||
rows = conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])', (UUID(chat_id), attachments)).fetchall()
|
||||
if len(rows) != len(attachments):
|
||||
return None
|
||||
scope = (UUID(chat_id), UUID(sender_id), key)
|
||||
row = conn.execute(
|
||||
"""INSERT INTO message_requests(chat_id, sender_id, key, text)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
ON CONFLICT (chat_id, sender_id, key) DO NOTHING
|
||||
RETURNING key""", (*scope, text)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
previous = conn.execute(
|
||||
"""SELECT m.* FROM message_requests r
|
||||
JOIN messages m ON m.id = r.message_id
|
||||
WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""",
|
||||
scope,
|
||||
).fetchone()
|
||||
if previous["text"] != text or previous["attachment_ids"] != attachments:
|
||||
return "conflict"
|
||||
return serialize_message(previous)
|
||||
message = conn.execute(
|
||||
"""INSERT INTO messages(id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES (%s, %s, %s, %s, %s) RETURNING *""",
|
||||
(uuid4(), UUID(chat_id), UUID(sender_id), text, attachments),
|
||||
).fetchone()
|
||||
conn.execute(
|
||||
"""UPDATE message_requests SET message_id=%s
|
||||
WHERE chat_id=%s AND sender_id=%s AND key=%s""",
|
||||
(message["id"], *scope),
|
||||
)
|
||||
return serialize_message(message)
|
||||
|
||||
|
||||
def list_messages(chat_id, limit, last_message_id=None):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
last_key = None
|
||||
|
||||
if last_message_id is not None:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT created_at, id
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
AND id = %s
|
||||
""",
|
||||
(UUID(chat_id), UUID(last_message_id)),
|
||||
)
|
||||
last_key = cur.fetchone()
|
||||
|
||||
if last_key is None:
|
||||
return None
|
||||
|
||||
# Разные формы SQL сохраняют индексный range scan и для generic plan.
|
||||
# В SQL подставляется только константа; значения остаются параметрами.
|
||||
cursor_condition = ""
|
||||
params = [UUID(chat_id)]
|
||||
if last_key is not None:
|
||||
cursor_condition = "AND (created_at, id) > (%s, %s)"
|
||||
params.extend([last_key["created_at"], last_key["id"]])
|
||||
params.append(limit + 1)
|
||||
cur.execute(
|
||||
f"""
|
||||
SELECT id, chat_id, sender_id, text, attachment_ids, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = %s
|
||||
{cursor_condition}
|
||||
ORDER BY created_at, id
|
||||
LIMIT %s
|
||||
""",
|
||||
params,
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
|
||||
return [serialize_message(row) for row in rows]
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Grade 4 orphan cleanup. Run periodically/manual; fails closed if DB is down.
|
||||
Only originals older than one hour, no committed metadata, and no active uploader.
|
||||
"""
|
||||
import json
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from .database import open_database, close_database, get_connection
|
||||
from .storage import s3, BUCKET
|
||||
|
||||
|
||||
def cleanup():
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
result = dict(scanned=0, fresh=0, referenced=0, busy=0, deleted=0)
|
||||
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='originals/'):
|
||||
for obj in page.get('Contents', []):
|
||||
result['scanned'] += 1
|
||||
if obj['LastModified'] >= cutoff:
|
||||
result['fresh'] += 1
|
||||
continue
|
||||
key = obj['Key']
|
||||
with get_connection() as conn:
|
||||
with conn.transaction():
|
||||
locked = conn.execute('SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0)) AS locked', (key,)).fetchone()['locked']
|
||||
if not locked:
|
||||
result['busy'] += 1
|
||||
elif conn.execute('SELECT 1 FROM attachments WHERE object_key=%s', (key,)).fetchone():
|
||||
result['referenced'] += 1
|
||||
else:
|
||||
s3.delete_object(Bucket=BUCKET, Key=key)
|
||||
result['deleted'] += 1
|
||||
return result
|
||||
|
||||
if __name__ == '__main__':
|
||||
open_database()
|
||||
try:
|
||||
print(json.dumps(cleanup()))
|
||||
finally:
|
||||
close_database()
|
||||
@@ -0,0 +1,14 @@
|
||||
"""Periodic grade 4 orphan cleanup, plus scripts/cleanup.sh for manual runs."""
|
||||
import json
|
||||
import time
|
||||
from .database import open_database
|
||||
from .cleanup import cleanup
|
||||
|
||||
open_database()
|
||||
while True:
|
||||
try:
|
||||
print(json.dumps(cleanup()), flush=True)
|
||||
except Exception:
|
||||
# Never log endpoint/credentials or underlying SDK exception contents.
|
||||
print('Cleanup unavailable; will retry in 300 seconds', flush=True)
|
||||
time.sleep(300)
|
||||
@@ -0,0 +1,48 @@
|
||||
import os
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
|
||||
def get_database_url() -> str:
|
||||
secret_file = os.getenv("DATABASE_URL_FILE")
|
||||
|
||||
if secret_file:
|
||||
return Path(secret_file).read_text().strip()
|
||||
|
||||
database_url = os.getenv("DATABASE_URL")
|
||||
|
||||
if database_url:
|
||||
return database_url
|
||||
|
||||
raise RuntimeError("Database connection is not configured")
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo=get_database_url(),
|
||||
min_size=1,
|
||||
max_size=10,
|
||||
timeout=5,
|
||||
open=False,
|
||||
kwargs={
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": 5,
|
||||
"options": "-c statement_timeout=5000",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def open_database():
|
||||
pool.open(wait=True, timeout=15)
|
||||
|
||||
|
||||
def close_database():
|
||||
pool.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def get_connection():
|
||||
with pool.connection() as connection:
|
||||
yield connection
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Isolated image decoder: bounded address space, CPU and parent wall timeout."""
|
||||
import resource
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
resource.setrlimit(resource.RLIMIT_AS, (512 * 1024**2, 512 * 1024**2))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (8, 8))
|
||||
from PIL import Image
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
try:
|
||||
with Image.open(sys.argv[1]) as image:
|
||||
if image.width * image.height > 40_000_000:
|
||||
sys.exit(13)
|
||||
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
|
||||
if actual != sys.argv[2] or getattr(image, 'n_frames', 1) != 1:
|
||||
sys.exit(15)
|
||||
image.verify()
|
||||
with Image.open(sys.argv[1]) as image:
|
||||
image.load()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
sys.exit(13)
|
||||
except Exception:
|
||||
sys.exit(15)
|
||||
+517
@@ -0,0 +1,517 @@
|
||||
"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили."""
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
from pathlib import Path
|
||||
import logging
|
||||
import re
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
|
||||
|
||||
from psycopg import OperationalError, Error as DatabaseError
|
||||
from psycopg_pool import PoolTimeout
|
||||
|
||||
from .auth import (
|
||||
create_token,
|
||||
format_datetime,
|
||||
hash_password,
|
||||
hash_token,
|
||||
parse_basic,
|
||||
parse_bearer,
|
||||
session_expiration,
|
||||
verify_password,
|
||||
)
|
||||
from .database import open_database, close_database, get_connection
|
||||
from .user_repository import (
|
||||
create_user as insert_user,
|
||||
create_session,
|
||||
find_session_user_cached,
|
||||
find_user_by_username,
|
||||
revoke_session,
|
||||
serialize_user,
|
||||
)
|
||||
|
||||
from .chat_repository import (
|
||||
create_chat as insert_chat,
|
||||
get_chat as find_chat,
|
||||
list_chats as find_chats,
|
||||
create_message as insert_message,
|
||||
list_messages as find_messages,
|
||||
)
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger", version="1.4.0")
|
||||
# Serve the mandatory contract in Swagger, including auth and request schemas.
|
||||
OPENAPI_SCHEMA = json.loads(
|
||||
(Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text()
|
||||
)
|
||||
app.openapi = lambda: OPENAPI_SCHEMA
|
||||
|
||||
INSTANCE_ID = str(uuid4())
|
||||
|
||||
logger = logging.getLogger("uvicorn.error")
|
||||
|
||||
ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
503: "unavailable",
|
||||
}
|
||||
|
||||
|
||||
def error_response(request, status_code, code, message):
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request.state.request_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# Логируем путь, метод и request_id, но НЕ тело сообщения.
|
||||
await run_in_threadpool(
|
||||
logger.error,
|
||||
"Unhandled error: %s %s request_id=%s",
|
||||
request.method,
|
||||
request.url.path,
|
||||
request.state.request_id,
|
||||
)
|
||||
response = error_response(
|
||||
request, 500, "internal_error", "Внутренняя ошибка сервера"
|
||||
)
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(request: Request, exc: StarletteHTTPException):
|
||||
response = error_response(
|
||||
request,
|
||||
exc.status_code,
|
||||
ERROR_CODES.get(exc.status_code, "invalid_request"),
|
||||
str(exc.detail),
|
||||
)
|
||||
if exc.headers:
|
||||
response.headers.update(exc.headers)
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(DatabaseError)
|
||||
@app.exception_handler(PoolTimeout)
|
||||
async def database_error_handler(request: Request, exc):
|
||||
return error_response(request, 503, "unavailable", "База данных временно недоступна")
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
||||
return error_response(request, 400, "invalid_request", "Некорректные данные запроса")
|
||||
|
||||
|
||||
async def read_json(request: Request):
|
||||
"""415 — только для неправильного Content-Type; 400 — для неверного JSON."""
|
||||
content_type = (
|
||||
request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
)
|
||||
if content_type != "application/json":
|
||||
raise HTTPException(415, "Неподдерживаемый Content-Type")
|
||||
|
||||
try:
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
if len(raw) + len(chunk) > 65536:
|
||||
raise HTTPException(413, "JSON слишком большой")
|
||||
raw.extend(chunk)
|
||||
data = json.loads(raw)
|
||||
except (ValueError, UnicodeError):
|
||||
raise HTTPException(400, "Неверный JSON") from None
|
||||
|
||||
# [], строка, число, true, false, null — это корректный JSON,
|
||||
# но НЕ объект, обязательный по контракту.
|
||||
if not isinstance(data, dict):
|
||||
raise HTTPException(400, "Ожидается JSON-объект")
|
||||
return data
|
||||
|
||||
|
||||
def get_current_user(request: Request):
|
||||
token = parse_bearer(request.headers.get("Authorization"))
|
||||
if token is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Требуется действующий Bearer-токен",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
try:
|
||||
user = find_session_user_cached(hash_token(token))
|
||||
except (OperationalError, PoolTimeout):
|
||||
raise HTTPException(
|
||||
503,
|
||||
"База данных временно недоступна",
|
||||
) from None
|
||||
if user is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Недействительная или истёкшая сессия",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
return user
|
||||
|
||||
|
||||
def get_limit(request: Request):
|
||||
"""Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500."""
|
||||
limit_raw = request.query_params.get("limit", "50")
|
||||
if not limit_raw.isascii() or not limit_raw.isdecimal():
|
||||
raise HTTPException(400, "Неверный limit")
|
||||
|
||||
significant = limit_raw.lstrip("0")
|
||||
if not significant or len(significant) > 3:
|
||||
raise HTTPException(400, "Неверный limit")
|
||||
|
||||
limit = int(significant) # Теперь строка содержит максимум 3 цифры.
|
||||
if not 1 <= limit <= 100:
|
||||
raise HTTPException(400, "limit должен быть от 1 до 100")
|
||||
return limit
|
||||
|
||||
|
||||
def encode_cursor(first_id, last_id):
|
||||
raw = f"{first_id}|{last_id}".encode("ascii")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(cursor):
|
||||
try:
|
||||
if not cursor or len(cursor) > 256:
|
||||
raise ValueError("empty or too long")
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
raw = base64.b64decode(
|
||||
padded, altchars=b"-_", validate=True
|
||||
).decode("ascii")
|
||||
first_id, last_id = raw.split("|")
|
||||
return str(UUID(first_id)), str(UUID(last_id))
|
||||
except (ValueError, UnicodeError, binascii.Error):
|
||||
raise HTTPException(400, "Неверный cursor") from None
|
||||
|
||||
|
||||
# ---- Health ----
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
try:
|
||||
with get_connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
from .storage import s3, BUCKET
|
||||
s3.list_objects_v2(Bucket=BUCKET, Prefix="originals/", MaxKeys=1)
|
||||
except (OperationalError, PoolTimeout):
|
||||
raise HTTPException(503, "База данных недоступна") from None
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
# ---- Users ----
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
data = await read_json(request)
|
||||
|
||||
if set(data) != {"username", "display_name", "password"}:
|
||||
raise HTTPException(400, "Неверные поля пользователя")
|
||||
|
||||
username = data["username"]
|
||||
display_name = data["display_name"]
|
||||
password = data["password"]
|
||||
|
||||
if not isinstance(username, str) or not re.fullmatch(
|
||||
r"[a-z0-9_]{3,32}", username
|
||||
):
|
||||
raise HTTPException(400, "Недопустимый username")
|
||||
|
||||
if (
|
||||
not isinstance(display_name, str)
|
||||
or not 1 <= len(display_name) <= 100
|
||||
or not display_name.strip()
|
||||
):
|
||||
raise HTTPException(400, "Недопустимое имя пользователя")
|
||||
|
||||
if not isinstance(password, str) or not 12 <= len(password) <= 128:
|
||||
raise HTTPException(400, "Пароль должен содержать 12–128 символов")
|
||||
|
||||
password_hash = await run_in_threadpool(hash_password, password)
|
||||
user = await run_in_threadpool(insert_user, username, display_name, password_hash)
|
||||
if user is None:
|
||||
raise HTTPException(409, "Username уже занят")
|
||||
|
||||
return user
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
def get_me(request: Request):
|
||||
return get_current_user(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
def login(request: Request):
|
||||
credentials = parse_basic(request.headers.get("Authorization"))
|
||||
|
||||
if credentials is None:
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Неверные учётные данные",
|
||||
headers={
|
||||
"WWW-Authenticate":
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
},
|
||||
)
|
||||
|
||||
username, password = credentials
|
||||
user_row = find_user_by_username(username)
|
||||
|
||||
if user_row is None or not verify_password(
|
||||
user_row["password_hash"], password
|
||||
):
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Неверные учётные данные",
|
||||
headers={
|
||||
"WWW-Authenticate":
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
},
|
||||
)
|
||||
|
||||
token = create_token()
|
||||
expires_at = session_expiration()
|
||||
|
||||
create_session(user_row["id"], hash_token(token), expires_at)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": format_datetime(expires_at),
|
||||
"user": serialize_user(user_row),
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def logout(request: Request):
|
||||
get_current_user(request)
|
||||
token = parse_bearer(request.headers.get("Authorization"))
|
||||
|
||||
from .session_cache import invalidate_session
|
||||
|
||||
if not revoke_session(hash_token(token)):
|
||||
raise HTTPException(
|
||||
401,
|
||||
"Недействительная сессия",
|
||||
headers={"WWW-Authenticate": 'Bearer realm="messenger"'},
|
||||
)
|
||||
|
||||
invalidate_session(hash_token(token))
|
||||
|
||||
from fastapi import Response
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
|
||||
# ---- Chats ----
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await run_in_threadpool(get_current_user, request)
|
||||
data = await read_json(request)
|
||||
|
||||
if set(data) != {"title", "member_ids"}:
|
||||
raise HTTPException(400, "Неверные поля чата")
|
||||
|
||||
title = data["title"]
|
||||
member_ids = data["member_ids"]
|
||||
|
||||
if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip():
|
||||
raise HTTPException(400, "Недопустимое название чата")
|
||||
|
||||
if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100:
|
||||
raise HTTPException(400, "Неверный список участников")
|
||||
|
||||
normalized_ids = []
|
||||
|
||||
for member_id in member_ids:
|
||||
if not isinstance(member_id, str):
|
||||
raise HTTPException(400, "Неверный ID участника")
|
||||
|
||||
try:
|
||||
normalized_ids.append(str(UUID(member_id)))
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверный UUID участника") from None
|
||||
|
||||
if len(normalized_ids) != len(set(normalized_ids)):
|
||||
raise HTTPException(400, "Участники повторяются")
|
||||
|
||||
if user["id"] not in normalized_ids:
|
||||
raise HTTPException(400, "Создатель должен быть участником")
|
||||
|
||||
chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids)
|
||||
|
||||
if chat is None:
|
||||
raise HTTPException(400, "Неизвестный участник")
|
||||
|
||||
return chat
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
def list_chats(request: Request):
|
||||
user = get_current_user(request)
|
||||
limit = get_limit(request)
|
||||
cursor = request.query_params.get("cursor")
|
||||
|
||||
last_chat_id = None
|
||||
|
||||
if cursor is not None:
|
||||
cursor_user_id, last_chat_id = decode_cursor(cursor)
|
||||
|
||||
if cursor_user_id != user["id"]:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
rows = find_chats(user["id"], limit, last_chat_id)
|
||||
|
||||
if rows is None:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = (
|
||||
encode_cursor(user["id"], page[-1]["id"])
|
||||
if has_more and page
|
||||
else None
|
||||
)
|
||||
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
def get_chat(chat_id: str, request: Request):
|
||||
user = get_current_user(request)
|
||||
|
||||
try:
|
||||
chat_id = str(UUID(chat_id))
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверный UUID чата") from None
|
||||
|
||||
chat = find_chat(chat_id, user["id"])
|
||||
|
||||
if chat is None:
|
||||
raise HTTPException(404, "Чат не найден")
|
||||
|
||||
return chat
|
||||
|
||||
|
||||
# ---- Messages ----
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def send_message(chat_id: str, request: Request):
|
||||
user = await run_in_threadpool(get_current_user, request)
|
||||
chat = await run_in_threadpool(get_chat, chat_id, request)
|
||||
data = await read_json(request)
|
||||
|
||||
if "text" not in data or set(data) - {"text", "attachment_ids"}:
|
||||
raise HTTPException(400, "Неверные поля сообщения")
|
||||
|
||||
text = data["text"]
|
||||
|
||||
attachment_ids = data.get("attachment_ids", [])
|
||||
if not isinstance(attachment_ids, list) or len(attachment_ids) > 10:
|
||||
raise HTTPException(400, "Неверный список вложений")
|
||||
try:
|
||||
if any(not isinstance(a, str) for a in attachment_ids):
|
||||
raise ValueError()
|
||||
attachment_ids = [str(UUID(a)) for a in attachment_ids]
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise ValueError()
|
||||
except ValueError:
|
||||
raise HTTPException(400, "Неверные ID вложений") from None
|
||||
if not isinstance(text, str) or len(text) > 4000 or (not text.strip() and not attachment_ids):
|
||||
raise HTTPException(400, "Недопустимый текст сообщения")
|
||||
|
||||
key = request.headers.get("Idempotency-Key", "")
|
||||
if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
|
||||
raise HTTPException(400, "Неверный Idempotency-Key")
|
||||
message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key, attachment_ids)
|
||||
if message == "conflict":
|
||||
raise HTTPException(409, "Ключ использован для другого сообщения")
|
||||
|
||||
if message is None:
|
||||
raise HTTPException(404, "Чат не найден")
|
||||
|
||||
return message
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
def list_messages(chat_id: str, request: Request):
|
||||
chat = get_chat(chat_id, request)
|
||||
limit = get_limit(request)
|
||||
cursor = request.query_params.get("cursor")
|
||||
|
||||
last_message_id = None
|
||||
|
||||
if cursor is not None:
|
||||
cursor_chat_id, last_message_id = decode_cursor(cursor)
|
||||
|
||||
if cursor_chat_id != chat["id"]:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
rows = find_messages(chat["id"], limit, last_message_id)
|
||||
|
||||
if rows is None:
|
||||
raise HTTPException(400, "Неверный cursor")
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = (
|
||||
encode_cursor(chat["id"], page[-1]["id"])
|
||||
if has_more and page
|
||||
else None
|
||||
)
|
||||
|
||||
return {"items": page, "next_cursor": next_cursor}
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
async def start_database_pool():
|
||||
await run_in_threadpool(open_database)
|
||||
|
||||
|
||||
@app.on_event("shutdown")
|
||||
async def stop_database_pool():
|
||||
await run_in_threadpool(close_database)
|
||||
|
||||
from .attachments import register
|
||||
register(app, get_current_user, get_chat)
|
||||
@@ -0,0 +1,16 @@
|
||||
"""One-shot migrations, serialized and recorded in PostgreSQL."""
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from .database import get_database_url
|
||||
|
||||
with psycopg.connect(get_database_url()) as conn:
|
||||
with conn.transaction():
|
||||
conn.execute("SELECT pg_advisory_xact_lock(310026)")
|
||||
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())")
|
||||
for path in sorted(Path("/app/db/migrations").glob("*.sql")):
|
||||
if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone():
|
||||
conn.execute(path.read_text())
|
||||
conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,))
|
||||
print("Applied", path.name, flush=True)
|
||||
else:
|
||||
print("Already applied", path.name, flush=True)
|
||||
@@ -0,0 +1,48 @@
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import redis
|
||||
|
||||
|
||||
def get_cache_url():
|
||||
secret_file = os.getenv("CACHE_URL_FILE")
|
||||
if secret_file:
|
||||
return Path(secret_file).read_text().strip()
|
||||
return os.getenv("CACHE_URL", "redis://redis:6379/0")
|
||||
|
||||
|
||||
client = redis.Redis.from_url(
|
||||
get_cache_url(),
|
||||
socket_connect_timeout=1,
|
||||
socket_timeout=1,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
|
||||
def get_cached_user(token_hash):
|
||||
try:
|
||||
value = client.get("session:" + token_hash)
|
||||
return json.loads(value) if value else None
|
||||
except (redis.RedisError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def cache_user(token_hash, user, ttl):
|
||||
if ttl <= 0:
|
||||
return
|
||||
try:
|
||||
client.setex(
|
||||
"session:" + token_hash,
|
||||
ttl,
|
||||
json.dumps(user),
|
||||
)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
|
||||
|
||||
def invalidate_session(token_hash):
|
||||
try:
|
||||
client.delete("session:" + token_hash)
|
||||
except redis.RedisError:
|
||||
pass
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Private S3 access; no automatic PUT retries, explicit bounded timeouts."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
def secret(name):
|
||||
return Path(os.environ[name + '_FILE']).read_text().strip()
|
||||
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
s3 = boto3.client(
|
||||
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
||||
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1},
|
||||
s3={'addressing_style': 'path'}),
|
||||
)
|
||||
@@ -0,0 +1,136 @@
|
||||
from uuid import uuid4
|
||||
|
||||
from psycopg.errors import UniqueViolation
|
||||
|
||||
from .database import get_connection
|
||||
|
||||
|
||||
def serialize_user(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": row["created_at"].isoformat(
|
||||
timespec="microseconds"
|
||||
).replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def create_user(username, display_name, password_hash):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
try:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO users (
|
||||
id, username, display_name, password_hash
|
||||
)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING id, username, display_name, created_at
|
||||
""",
|
||||
(uuid4(), username, display_name, password_hash),
|
||||
)
|
||||
return serialize_user(cur.fetchone())
|
||||
except UniqueViolation:
|
||||
conn.rollback()
|
||||
return None
|
||||
|
||||
|
||||
def find_user_by_username(username):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, username, display_name,
|
||||
password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = %s
|
||||
""",
|
||||
(username,),
|
||||
)
|
||||
return cur.fetchone()
|
||||
|
||||
|
||||
def create_session(user_id, token_hash, expires_at):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO sessions (
|
||||
id, user_id, token_hash, expires_at
|
||||
)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
""",
|
||||
(uuid4(), user_id, token_hash, expires_at),
|
||||
)
|
||||
|
||||
|
||||
def revoke_session(token_hash):
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET revoked_at = NOW()
|
||||
WHERE token_hash = %s
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > NOW()
|
||||
RETURNING id
|
||||
""",
|
||||
(token_hash,),
|
||||
)
|
||||
return cur.fetchone() is not None
|
||||
|
||||
|
||||
def find_session_user_cached(token_hash):
|
||||
from datetime import datetime, timezone
|
||||
from .session_cache import cache_user, get_cached_user
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
s.user_id,
|
||||
s.expires_at
|
||||
FROM sessions s
|
||||
WHERE s.token_hash = %s
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > NOW()
|
||||
""",
|
||||
(token_hash,),
|
||||
)
|
||||
session = cur.fetchone()
|
||||
|
||||
if session is None:
|
||||
return None
|
||||
|
||||
cached = get_cached_user(token_hash)
|
||||
|
||||
if cached is not None and str(cached.get("id")) == str(session["user_id"]):
|
||||
return cached
|
||||
|
||||
with get_connection() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, username, display_name, created_at
|
||||
FROM users
|
||||
WHERE id = %s
|
||||
""",
|
||||
(session["user_id"],),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
user = serialize_user(row)
|
||||
ttl = int(
|
||||
(session["expires_at"] - datetime.now(timezone.utc)).total_seconds()
|
||||
)
|
||||
|
||||
if ttl > 0:
|
||||
cache_user(token_hash, user, ttl)
|
||||
|
||||
return user
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Stop exactly the API that returned upload 201, download via survivor; repeat both."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
from faults import compose, ready
|
||||
|
||||
|
||||
def mappings():
|
||||
result = {}
|
||||
for container in compose('ps', '-q', 'api').split():
|
||||
identity = subprocess.check_output(['docker', 'exec', container, 'python', '-c', 'import urllib.request;print(urllib.request.urlopen("http://localhost:8080/health/live").headers["X-Instance-Id"])'], text=True).strip()
|
||||
result[identity] = container
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=20)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
image=(Path(__file__).resolve().parents[1]/'tests/fixtures/sample.png').read_bytes()
|
||||
results=[]
|
||||
stopped=set()
|
||||
try:
|
||||
for _ in range(2):
|
||||
mapping=mappings()
|
||||
for attempt in range(20):
|
||||
response=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,actor=alice,raw=image,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
instance=response['headers']['X-Instance-Id']
|
||||
container=mapping[instance]
|
||||
if container not in stopped:
|
||||
break
|
||||
else:
|
||||
raise RuntimeError('could not reach other uploader')
|
||||
att=response['json']
|
||||
env=dict(os.environ,TARGET_INSTANCE_ID=instance)
|
||||
subprocess.run(['scripts/stop-one.sh'],env=env,check=True)
|
||||
status=subprocess.check_output(['docker','inspect','--format','{{.State.Running}}',container],text=True).strip()
|
||||
check(status=='false','uploader still running')
|
||||
deadline=time.monotonic()+10
|
||||
while True:
|
||||
try:
|
||||
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
break
|
||||
except AssertionError:
|
||||
if time.monotonic()>deadline:
|
||||
raise
|
||||
time.sleep(.2)
|
||||
check(download['headers']['X-Instance-Id']!=instance,'served by stopped uploader')
|
||||
check(download['bytes']==image,'original changed after uploader stop')
|
||||
results.append({'uploader_instance':instance,'stopped_container':container[:12],'survivor_instance':download['headers']['X-Instance-Id'],'sha256':hashlib.sha256(download['bytes']).hexdigest(),'original_identical':True})
|
||||
stopped.add(container)
|
||||
compose('start','api')
|
||||
time.sleep(4)
|
||||
ready(c)
|
||||
print(json.dumps({'passed':True,'both_uploaders_tested':len(stopped),'scenarios':results},indent=2))
|
||||
finally:
|
||||
compose('start','api')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,23 @@
|
||||
"""Verify every committed original against PostgreSQL, without printing keys/tokens."""
|
||||
import json
|
||||
from faults import remote
|
||||
|
||||
code = '''from src.database import open_database,get_connection
|
||||
from src.storage import s3,BUCKET
|
||||
import hashlib,json
|
||||
open_database()
|
||||
with get_connection() as conn:
|
||||
rows=conn.execute('SELECT object_key,size_bytes,sha256 FROM attachments').fetchall()
|
||||
checked=total=0
|
||||
for row in rows:
|
||||
body=s3.get_object(Bucket=BUCKET,Key=row['object_key'])['Body']
|
||||
digest=hashlib.sha256(); size=0
|
||||
try:
|
||||
for chunk in body.iter_chunks(65536):
|
||||
size+=len(chunk); digest.update(chunk)
|
||||
finally:
|
||||
body.close()
|
||||
assert size==row['size_bytes'] and digest.hexdigest()==row['sha256'], 'original mismatch'
|
||||
checked+=1; total+=size
|
||||
print(json.dumps({'verified_originals':checked,'verified_bytes':total,'all_checksums_match':True}))'''
|
||||
print(json.dumps(json.loads(remote(code)),indent=2))
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Real lab4-only fault injection. Restores services/triggers in finally.
|
||||
Operator invokes explicitly; never removes volumes or user objects.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = ['docker', 'compose', '--env-file', str(ROOT / '.env')]
|
||||
|
||||
|
||||
def compose(*args, data=None):
|
||||
result = subprocess.run(COMPOSE + list(args), input=data, text=True, capture_output=True, cwd=ROOT)
|
||||
if result.returncode:
|
||||
raise RuntimeError('compose action failed: ' + ' '.join(args[:2]))
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def sql(statement):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', data=statement)
|
||||
|
||||
|
||||
def remote(code, data=None):
|
||||
return compose('exec', '-T', 'api', 'python', '-c', code, data=data)
|
||||
|
||||
|
||||
def ready(c):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call('GET', '/health/ready')
|
||||
return
|
||||
except AssertionError:
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('readiness timeout')
|
||||
|
||||
|
||||
S3_SUMMARY = '''from src.storage import s3, BUCKET
|
||||
import json
|
||||
objects = [o['Key'] for p in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix='originals/') for o in p.get('Contents', [])]
|
||||
print(json.dumps(objects))'''
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=40)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
attachment, image = c.attachment(alice, chat)
|
||||
passed = []
|
||||
# Only our unique filename will trigger a database failure AFTER a real S3 PUT.
|
||||
filename = 'fault-' + chat['id'] + '.png'
|
||||
trigger = f'''CREATE OR REPLACE FUNCTION student_fail_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN IF NEW.filename = '{filename}' THEN RAISE EXCEPTION 'synthetic attachment failure'; END IF; RETURN NEW; END $$;
|
||||
CREATE TRIGGER student_fail_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_fail_attachment();'''
|
||||
try:
|
||||
before = set(json.loads(remote(S3_SUMMARY)))
|
||||
referenced = set(json.loads(remote('''from src.database import open_database, get_connection
|
||||
open_database()
|
||||
import json
|
||||
with get_connection() as conn:
|
||||
print(json.dumps([r['object_key'] for r in conn.execute('SELECT object_key FROM attachments').fetchall()]))''')))
|
||||
sql(trigger)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, 503, actor=alice, raw=image, headers={'Content-Type': 'image/png'})
|
||||
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
|
||||
after = set(json.loads(remote(S3_SUMMARY)))
|
||||
orphan = after - before
|
||||
check(len(orphan) == 1, 'did not observe a real orphan after DB rollback')
|
||||
passed.append('S3_PUT_then_DB_failure_returns_503_with_real_orphan')
|
||||
# Fresh objects survive a real cleanup. Clock advance is confined to this test
|
||||
# process: production grace period and object timestamps are not changed.
|
||||
result = json.loads(remote('''from src.database import open_database
|
||||
open_database()
|
||||
from src.cleanup import cleanup
|
||||
import json
|
||||
print(json.dumps(cleanup()))'''))
|
||||
check(result['deleted'] == 0 and result['fresh'] > 0, 'fresh object deleted')
|
||||
passed.append('fresh_orphan_survives_one_hour_grace')
|
||||
result = json.loads(remote('''from src.database import open_database
|
||||
open_database()
|
||||
import src.cleanup as cleaner
|
||||
from datetime import datetime, timezone, timedelta
|
||||
class Future(datetime):
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return datetime.now(timezone.utc) + timedelta(hours=2)
|
||||
cleaner.datetime = Future
|
||||
import json
|
||||
print(json.dumps(cleaner.cleanup()))'''))
|
||||
check(result['deleted'] >= 1 and result['referenced'] > 0, 'cleanup failed to preserve references / delete orphan')
|
||||
final = set(json.loads(remote(S3_SUMMARY)))
|
||||
check(not orphan & final and referenced <= final, 'cleanup lost used data')
|
||||
passed.append('expired_orphan_deleted_referenced_objects_preserved_test_clock')
|
||||
repeat = json.loads(remote('''from src.database import open_database
|
||||
open_database()
|
||||
from src.cleanup import cleanup
|
||||
import json
|
||||
print(json.dumps(cleanup()))'''))
|
||||
check(repeat['deleted'] == 0, 'cleanup not repeatable')
|
||||
passed.append('cleanup_repeatable')
|
||||
# API credentials: anonymous GET/list denied, bucket policy/admin denied.
|
||||
privacy = json.loads(remote('''from src.storage import s3, BUCKET
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
import boto3, json, urllib.request, urllib.error
|
||||
key=s3.list_objects_v2(Bucket=BUCKET,Prefix='originals/',MaxKeys=1)['Contents'][0]['Key']
|
||||
result=[]
|
||||
for suffix in ('/'+BUCKET+'/'+key, '/'+BUCKET+'?list-type=2'):
|
||||
try:
|
||||
urllib.request.urlopen('http://s3:9000'+suffix,timeout=3)
|
||||
result.append(200)
|
||||
except urllib.error.HTTPError as e:
|
||||
result.append(e.code)
|
||||
try:
|
||||
s3.get_bucket_policy(Bucket=BUCKET)
|
||||
result.append(200)
|
||||
except ClientError as e:
|
||||
result.append(e.response['ResponseMetadata']['HTTPStatusCode'])
|
||||
print(json.dumps(result))'''))
|
||||
check(privacy == [403, 403, 403], 'bucket or API credentials overexposed')
|
||||
passed.append('anonymous_GET_and_LIST_denied_API_cannot_read_policy')
|
||||
# Probe API itself, not a load-balancer-generated failure.
|
||||
request = json.dumps({'token': alice['token'], 'chat': chat['id'], 'image': base64.b64encode(image).decode()})
|
||||
probe = '''import sys,json,base64,urllib.request,urllib.error
|
||||
x=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/'+x['chat']+'/attachments?filename=outage.png',data=base64.b64decode(x['image']),headers={'Authorization':'Bearer '+x['token'],'Content-Type':'image/png'},method='POST')
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=20)
|
||||
except urllib.error.HTTPError as e:
|
||||
response=e
|
||||
print(json.dumps({'status':response.code,'body':json.loads(response.read()),'request_id':bool(response.headers.get('X-Request-Id')),'instance_id':bool(response.headers.get('X-Instance-Id'))}))'''
|
||||
for service in ('s3', 'postgres'):
|
||||
compose('stop', service)
|
||||
try:
|
||||
failure = json.loads(remote(probe, data=request))
|
||||
check(failure['status'] == 503 and failure['request_id'] and failure['instance_id'], service + ' outage mishandled')
|
||||
check(failure['body']['error']['code'] == 'unavailable', 'wrong error format')
|
||||
finally:
|
||||
compose('start', service)
|
||||
ready(c)
|
||||
passed.append(service + '_outage_API_503')
|
||||
# An actual disconnect cannot create an object because PUT follows full validation.
|
||||
before = set(json.loads(remote(S3_SUMMARY)))
|
||||
import socket
|
||||
from urllib.parse import urlsplit
|
||||
url = urlsplit(c.base)
|
||||
con = socket.create_connection((url.hostname, url.port))
|
||||
con.sendall((f'POST /api/v1/chats/{chat["id"]}/attachments?filename=aborted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {alice["token"]}\r\nContent-Type: image/png\r\nContent-Length: {len(image)}\r\n\r\n').encode() + image[:100])
|
||||
con.close()
|
||||
time.sleep(2)
|
||||
check(before == set(json.loads(remote(S3_SUMMARY))), 'aborted upload left an S3 object')
|
||||
passed.append('interrupted_upload_no_S3_object')
|
||||
# Kill the process while the metadata INSERT sleeps after PUT.
|
||||
sql(f'''CREATE OR REPLACE FUNCTION student_fail_attachment() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN IF NEW.filename = '{filename}' THEN PERFORM pg_sleep(20); END IF; RETURN NEW; END $$;
|
||||
CREATE TRIGGER student_fail_attachment BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION student_fail_attachment();''')
|
||||
before = set(json.loads(remote(S3_SUMMARY)))
|
||||
with ThreadPoolExecutor(max_workers=1) as pool:
|
||||
future = pool.submit(c.call, 'POST', '/api/v1/chats/'+chat['id']+'/attachments?filename='+filename, 201, actor=alice, raw=image, headers={'Content-Type':'image/png'})
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
observed = set(json.loads(remote(S3_SUMMARY))) - before
|
||||
if observed:
|
||||
break
|
||||
time.sleep(.2)
|
||||
check(len(observed) == 1, 'PUT not observed before crash')
|
||||
compose('kill', '-s', 'SIGKILL', 'api')
|
||||
try:
|
||||
future.result()
|
||||
raise RuntimeError('crashed request unexpectedly completed')
|
||||
except AssertionError:
|
||||
pass
|
||||
compose('up', '-d', '--no-deps', 'api')
|
||||
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
|
||||
ready(c)
|
||||
check(int(sql("SELECT count(*) FROM attachments WHERE filename='"+filename+"';")) == 0, 'crash committed metadata')
|
||||
check(observed <= set(json.loads(remote(S3_SUMMARY))), 'crash orphan not observed')
|
||||
passed.append('API_SIGKILL_after_PUT_before_COMMIT_orphan_recoverable')
|
||||
crash_cleanup = json.loads(remote('''from src.database import open_database
|
||||
open_database()
|
||||
import src.cleanup as cleaner
|
||||
from datetime import datetime, timezone, timedelta
|
||||
class Future(datetime):
|
||||
@classmethod
|
||||
def now(cls, tz=None):
|
||||
return datetime.now(timezone.utc) + timedelta(hours=2)
|
||||
cleaner.datetime = Future
|
||||
import json
|
||||
print(json.dumps(cleaner.cleanup()))'''))
|
||||
check(crash_cleanup['deleted'] >= 1, 'crash orphan cleanup did not run')
|
||||
check(not observed & set(json.loads(remote(S3_SUMMARY))), 'crash orphan not removed')
|
||||
passed.append('API_crash_orphan_actually_cleaned_after_test_clock_grace')
|
||||
got = c.call('GET', '/api/v1/attachments/'+attachment['id']+'/content', actor=bob)['bytes']
|
||||
check(got == image, 'confirmed attachment lost during failures')
|
||||
passed.append('confirmed_original_survives_faults')
|
||||
print(json.dumps({'passed': passed, 'count': len(passed), 'privacy_status_codes': privacy, 'cleanup_expired_result': result, 'grace_test': 'test process clock +2h; production grace remains 1h'}, indent=2))
|
||||
finally:
|
||||
compose('start', 'postgres', 's3')
|
||||
compose('up', '-d', '--no-deps', 'api')
|
||||
sql('DROP TRIGGER IF EXISTS student_fail_attachment ON attachments; DROP FUNCTION IF EXISTS student_fail_attachment();')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,3 @@
|
||||
sample.jpg — синтетический RGB JPEG 32×24, однотонный, создан Pillow 11.2.1
|
||||
в контейнере lab4. Без персональных данных; проверяет второй разрешённый формат.
|
||||
Публичный PNG fixture не изменён.
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 643 B |
@@ -0,0 +1,104 @@
|
||||
"""Black-box grade 4 checks; public tests stay untouched. Standard library only."""
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote, urlsplit
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def png(width, height, pixels=None):
|
||||
def chunk(kind, data):
|
||||
return struct.pack('!I', len(data)) + kind + data + struct.pack('!I', zlib.crc32(kind + data))
|
||||
raw = pixels if pixels is not None else b'\0' * ((width * 3 + 1) * height)
|
||||
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('!IIBBBBB', width, height, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b'')
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=45)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
base = '/api/v1/chats/' + chat['id']
|
||||
path = base + '/attachments?filename='
|
||||
image = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||||
passed = []
|
||||
def upload(name, data, status, mime='image/png'):
|
||||
return c.call('POST', path + quote(name, safe=''), status, actor=alice, raw=data, headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None)
|
||||
cases = [('bytes_over_10MiB', 'large.png', b'x' * (10485760 + 1), 413, 'image/png'),
|
||||
('pixels_over_40M', 'bomb.png', png(40000001, 1, b'\0'), 413, 'image/png'),
|
||||
('broken_png', 'broken.png', image[:70], 415, 'image/png'),
|
||||
('broken_jpeg', 'broken.jpg', b'\xff\xd8\xff\xe0truncated', 415, 'image/jpeg'),
|
||||
('mime_mismatch', 'wrong.jpg', image, 415, 'image/jpeg'),
|
||||
('invalid_format', 'bad.png', b'plain text', 415, 'image/png'),
|
||||
('multipart_rejected', 'bad.png', image, 415, 'multipart/form-data')]
|
||||
for name, filename, data, status, mime in cases:
|
||||
upload(filename, data, status, mime)
|
||||
passed.append(name)
|
||||
for filename in ('../x.png', 'x\\y.png', 'x\0.png', 'x\n.png', '\x7f.png', '', 'x' * 256):
|
||||
upload(filename, image, 400)
|
||||
passed.append('dangerous_filenames')
|
||||
att, data = c.attachment(alice, chat)
|
||||
for actor in (alice, bob):
|
||||
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)['bytes']
|
||||
check(hashlib.sha256(got).hexdigest() == att['sha256'] and got == data, 'checksum mismatch')
|
||||
passed.append('original_checksum_and_shared_access')
|
||||
jpeg = (Path(__file__).resolve().parent / 'fixtures/sample.jpg').read_bytes()
|
||||
jpeg_att = upload('original.jpg', jpeg, 201, 'image/jpeg')['json']
|
||||
downloaded = c.call('GET', '/api/v1/attachments/' + jpeg_att['id'] + '/content', actor=bob)
|
||||
check(downloaded['bytes'] == jpeg and downloaded['headers'].get_content_type() == 'image/jpeg', 'JPEG original changed')
|
||||
check(jpeg_att['sha256'] == hashlib.sha256(jpeg).hexdigest(), 'JPEG checksum mismatch')
|
||||
passed.append('JPEG_success_and_checksum')
|
||||
for suffix in ('', '/content'):
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 401)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve)
|
||||
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
|
||||
passed.append('download_auth_and_acl')
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
foreign = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'private', 'member_ids': [eve['id']]}, schema='Chat')['json']
|
||||
foreign_att, _ = c.attachment(eve, foreign)
|
||||
for ids in ([str(uuid.uuid4())], [foreign_att['id']]):
|
||||
c.call('POST', base + '/messages', 404, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
passed.append('unknown_foreign_and_other_chat_attachment')
|
||||
for ids in ([att['id']] * 2, [str(uuid.uuid4()) for _ in range(11)], 'invalid', [None]):
|
||||
c.call('POST', base + '/messages', 400, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
passed.append('attachment_id_validation')
|
||||
key = uuid.uuid4().hex
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
messages = list(pool.map(lambda _: Client(c.base, timeout=30).message(alice, chat, '', key=key, attachments=[att['id']])['json'], range(10)))
|
||||
check(all(x == messages[0] for x in messages), 'concurrent replay differs')
|
||||
c.call('POST', base + '/messages', 409, actor=alice, body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': key})
|
||||
key = uuid.uuid4().hex
|
||||
first = c.message(alice, chat, 'old text', key=key)['json']
|
||||
check(first == c.message(alice, chat, 'old text', key=key, attachments=[])['json'], 'default [] differs')
|
||||
passed.append('attachment_idempotency_across_replicas')
|
||||
# Actual chunked request without Content-Length exercises incremental byte limit.
|
||||
url = urlsplit(c.base)
|
||||
con = http.client.HTTPConnection(url.hostname, url.port, timeout=30)
|
||||
con.request('POST', path + 'chunked.png', body=iter([b'x' * 65536] * 161), headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'}, encode_chunked=True)
|
||||
response = con.getresponse()
|
||||
check(response.status == 413, 'chunked limit failed')
|
||||
response.read()
|
||||
con.close()
|
||||
passed.append('chunked_body_limit')
|
||||
# Close mid-body before image validation / S3 PUT.
|
||||
con = socket.create_connection((url.hostname, url.port))
|
||||
headers = f'POST {path}interrupted.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {alice["token"]}\r\nContent-Type: image/png\r\nContent-Length: {len(image)}\r\n\r\n'.encode()
|
||||
con.sendall(headers + image[:100])
|
||||
con.close()
|
||||
time.sleep(1)
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
passed.append('client_disconnect_server_survives')
|
||||
output = {'passed': passed, 'count': len(passed), 'note': 'Fault/orphan checks are separate in faults.py'}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Ten barrier-synchronized uploads + cgroup v2 memory/CPU sampling, lab4 only."""
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from negative import png
|
||||
from client import Client
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://localhost:8084'), timeout=45)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
# Random RGB is poorly compressible; ~7.3 MiB per original, no fixture in Git.
|
||||
width = height = 1600
|
||||
raw = b''.join(b'\0' + os.urandom(width * 3) for _ in range(height))
|
||||
image = png(width, height, raw)
|
||||
del raw
|
||||
compose = ['docker', 'compose', '--env-file', '.env']
|
||||
containers = subprocess.check_output(compose + ['ps', '-q', 'api'], text=True).split()
|
||||
code = '''import json,time
|
||||
from pathlib import Path
|
||||
root=Path('/sys/fs/cgroup')
|
||||
def cpu():
|
||||
return int(dict(line.split() for line in (root/'cpu.stat').read_text().splitlines())['usage_usec'])
|
||||
start=time.monotonic(); first=cpu(); values=[]
|
||||
while time.monotonic()-start < 20:
|
||||
values.append(int((root/'memory.current').read_text()))
|
||||
time.sleep(.02)
|
||||
print(json.dumps({'sample_count':len(values),'sampled_baseline_bytes':values[0],'sampled_peak_bytes':max(values),'cgroup_lifetime_peak_bytes':int((root/'memory.peak').read_text()),'elapsed_seconds':time.monotonic()-start,'cpu_usage_seconds':(cpu()-first)/1e6}))'''
|
||||
samplers = [subprocess.Popen(['docker', 'exec', name, 'python', '-c', code], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) for name in containers]
|
||||
time.sleep(1)
|
||||
barrier = threading.Barrier(10)
|
||||
def worker(index):
|
||||
local = Client(c.base, timeout=45)
|
||||
barrier.wait()
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = local.call('POST', '/api/v1/chats/'+chat['id']+'/attachments?filename=load-'+str(index)+'.png', 201, actor=alice, raw=image, headers={'Content-Type':'image/png'}, schema='Attachment')
|
||||
return {'ok': True, 'seconds': time.monotonic()-started, 'instance': response['headers']['X-Instance-Id']}
|
||||
except Exception as exc:
|
||||
return {'ok': False, 'seconds': time.monotonic()-started, 'error': type(exc).__name__}
|
||||
started = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
results = list(pool.map(worker, range(10)))
|
||||
elapsed = time.monotonic() - started
|
||||
metrics = []
|
||||
for name, proc in zip(containers, samplers):
|
||||
out, err = proc.communicate(timeout=30)
|
||||
if proc.returncode:
|
||||
raise RuntimeError('cgroup sampler failed')
|
||||
record = json.loads(out)
|
||||
record['container'] = name[:12]
|
||||
record['average_cpu_percent_one_core'] = 100 * record['cpu_usage_seconds'] / record['elapsed_seconds']
|
||||
metrics.append(record)
|
||||
latencies = sorted(x['seconds'] for x in results if x['ok'])
|
||||
successes = len(latencies)
|
||||
report = {'utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), 'concurrency': 10, 'requests': 10,
|
||||
'original_bytes': len(image), 'dimensions': [width,height], 'successful_requests': successes,
|
||||
'failed_requests': 10-successes, 'elapsed_seconds': elapsed, 'successful_rps': successes/elapsed,
|
||||
'latency_success_seconds': {p: latencies[max(0, math.ceil(q*len(latencies))-1)] if latencies else None for p,q in [('p50',.5),('p95',.95),('p99',.99)]},
|
||||
'api_cgroups': metrics, 'requests_detail': results,
|
||||
'limitations': 'One local closed-loop batch; client overhead; no capacity claim. cgroup memory.current sampled every 20ms includes decoder and tmpfs/page cache; may miss shorter peaks. memory.peak is lifetime, includes earlier work. CPU averaged over 20s sampling window includes idle time. Per-API peaks need not be simultaneous.',
|
||||
'docker_host': subprocess.check_output(['docker','info','--format','{{.Architecture}} CPUs={{.NCPU}} RAM={{.MemTotal}}'],text=True).strip()}
|
||||
text=json.dumps(report,indent=2)
|
||||
Path('evidence/upload-load.json').write_text(text+'\n')
|
||||
print(text)
|
||||
if successes != 10:
|
||||
raise SystemExit(1)
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user