ЛР 4 — БПИ2303 — Лукьянов — на 4 #100
@@ -18,3 +18,6 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
.superpowers
|
||||
.pytest_cache
|
||||
|
||||
+5
-19
@@ -1,20 +1,6 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Created by scripts/bootstrap.sh; .env and secrets/ are never committed.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Compose supplies DATABASE_URL_FILE, CACHE_URL_FILE and S3 credentials *_FILE.
|
||||
# S3_ENDPOINT=http://minio:9000, S3_REGION=us-east-1, S3_BUCKET=messenger-attachments.
|
||||
# Contract limits:10MiB and40M pixels. Six uploads and one decoder per API.
|
||||
# Cleanup uses grace300s, interval60s. Only Nginx publishes127.0.0.1:8080.
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
name: Application acceptance
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
cluster:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- run: make up
|
||||
- run: make check test test-student
|
||||
- run: make test-fresh
|
||||
- run: make test-cluster
|
||||
- run: make test-attachments
|
||||
- run: make uploads
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: lab4-evidence
|
||||
path: evidence/
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
FROM python:3.13.5-slim
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements-dev.txt requirements.lock ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
|
||||
COPY src/ src/
|
||||
COPY migrations/ migrations/
|
||||
COPY student-tests/ student-tests/
|
||||
COPY scripts/ scripts/
|
||||
COPY tests/ tests/
|
||||
COPY contracts/ contracts/
|
||||
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
|
||||
ARG APP_RELEASE=lab4-v1
|
||||
ENV APP_RELEASE=${APP_RELEASE}
|
||||
LABEL org.opencontainers.image.version=${APP_RELEASE}
|
||||
USER messenger
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
|
||||
@@ -28,3 +28,42 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down test-student test-cluster rolling
|
||||
bootstrap:
|
||||
./scripts/bootstrap.sh
|
||||
up: bootstrap
|
||||
docker compose build api minio minio-init
|
||||
docker compose up -d --wait --wait-timeout 120
|
||||
$(PYTHON) scripts/cluster.py wait
|
||||
down:
|
||||
docker compose down
|
||||
test-student:
|
||||
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
test-cluster:
|
||||
$(PYTHON) scripts/verify_cluster.py
|
||||
rolling:
|
||||
$(PYTHON) scripts/cluster.py rolling
|
||||
|
||||
.PHONY: test-fresh load-evidence verify-persistence verify-failover
|
||||
test-fresh:
|
||||
./scripts/check_fresh_migrations.sh
|
||||
load-evidence:
|
||||
$(PYTHON) scripts/capture_load.py
|
||||
verify-persistence:
|
||||
$(PYTHON) scripts/verify_cluster.py persistence
|
||||
verify-failover:
|
||||
$(PYTHON) scripts/verify_cluster.py public-failover
|
||||
$(PYTHON) scripts/verify_cluster.py failover
|
||||
|
||||
.PHONY: validate-container
|
||||
validate-container:
|
||||
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
|
||||
.PHONY: test-attachments uploads cleanup-dry-run
|
||||
test-attachments:
|
||||
$(PYTHON) scripts/verify_attachments.py
|
||||
uploads:
|
||||
$(PYTHON) scripts/capture_uploads.py
|
||||
cleanup-dry-run:
|
||||
docker compose run --rm --no-deps cleanup python -m messenger.cleanup --dry-run
|
||||
|
||||
@@ -1,37 +1,149 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Лукьянов К.Р. БПИ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не заявлен)
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; документация [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Pillow Image](https://pillow.readthedocs.io/en/stable/reference/Image.html), [Boto3 configuration](https://docs.aws.amazon.com/boto3/latest/guide/configuration.html), [MinIO IAM](https://github.com/minio/minio/blob/master/docs/multi-user/README.md), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker с Compose v2, Python ≥ 3.9 на хосте для стандартных тестов/оркестрации, Make, свободный localhost:8080. Проверено на Docker Desktop 29.8.2, macOS/Apple M4 Pro. Go и Python-зависимости приложения устанавливаются внутри сборочных контейнеров. Для первой сборки нужен доступ к GitHub и Go/Python module repositories.
|
||||
|
||||
```sh
|
||||
git clone <fork-url> && cd <каталог-fork-лабораторной-4>
|
||||
make up # secrets → сборка API/MinIO/mc → migrate/init → api×2/cleanup/Nginx
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
docker compose ps --all # API×2/PG/Redis/MinIO/cleanup/LB работают; migrate и minio-init — Exited(0)
|
||||
|
||||
make check
|
||||
make validate-container
|
||||
make test # публичный smoke lab4 через Nginx
|
||||
make test-student # 21 собственный интеграционный тест
|
||||
make test-fresh # миграции 001–003, новый bucket и тесты на отдельных пустых volumes
|
||||
make test-attachments # private bucket, upload→отказ API→download, обрыв, crash+cleanup, отказ S3
|
||||
make uploads # 10 одновременных загрузок с измерением пика RAM
|
||||
make test-cluster # сохранённые проверки общей сессии/idempotency/restart/failover/SIGTERM/rolling
|
||||
```
|
||||
|
||||
`make up` выполняет bootstrap, `docker compose build api minio minio-init`, затем `up --wait` и ожидание двух API-alias через LB. При повторной сборке Go/Python-слои используются из кэша. Сценарии отказов и замер памяти запускать по одному: они останавливают/пересоздают контейнеры. `make uploads` пересоздаёт только API, чтобы сбросить kernel memory high-water marks; volumes и данные сохраняются.
|
||||
|
||||
**Почему MinIO собирается из исходников:** проверенные release-образы в Docker Hub/Quay недоступны, официальный архив бинарников ответил 410. В `infra/minio/Dockerfile` собираются официальные теги и проверяется точный commit: MinIO `RELEASE.2025-04-22T22-12-26Z` / `0d7408fc9969caf07de6a8c3a84f9fbb10a6739e`, mc `RELEASE.2025-04-16T18-13-26Z` / `b00526b153a31b36767991a4f5ce2cced435ee8e`. Builder — `golang:1.24.2-bookworm`, зависимости сверяются по upstream go.sum. Runtime-образы локальные `mtusi-lab4-minio:2025-04-22` и `mtusi-lab4-mc:2025-04-16`; скачивать готовый сервер преподавателя не требуется.
|
||||
|
||||
`bootstrap.sh` однократно создаёт пароль БД, root-credentials MinIO и отдельные API S3 credentials в `secrets/`; повтор их не меняет. Каталог 0700 на хосте, файлы доступны нужному uid через Compose secrets. Root S3 secrets получает только MinIO и minio-init; API/cleanup получают ограниченного пользователя. `.env`, secrets и volumes не входят в Git/build context. Учебные данные и credentials lab3 не копировались.
|
||||
|
||||
Безопасная остановка: `make down`, именованные volumes `postgres_data`, `redis_data`, `minio_data` сохраняются. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Restart hook volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. После обычных тестов синтетические пользователи/сообщения/вложения остаются — delete не входит в обязательный API.
|
||||
|
||||
Отдельные команды:
|
||||
|
||||
```sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# После отдельного публичного failover восстановить остановленную реплику:
|
||||
docker compose up -d --no-recreate --wait --wait-timeout 90
|
||||
python3 scripts/cluster.py wait
|
||||
|
||||
make cleanup-dry-run
|
||||
# Явная безопасная очистка с обычным grace 300s:
|
||||
docker compose run --rm --no-deps cleanup python -m messenger.cleanup --apply
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
клиент → Nginx → API ×2 ──→ PostgreSQL: пользователи/чаты/сообщения/сессии/metadata/связи
|
||||
├─→ Redis: кэш публичного профиля
|
||||
└─→ MinIO: оригинальные байты, приватный bucket
|
||||
|
||||
migrate → PostgreSQL minio-init → bucket/user/policy
|
||||
cleanup → S3 listing + PostgreSQL recheck → удаление только старых неучтённых объектов
|
||||
```
|
||||
|
||||
- **Перенос из lab3.** Сохранены Nginx, два экземпляра одного service api, общая PostgreSQL, Argon2id/Basic→Bearer, Redis cache-aside профиля, keyset-пагинация, X-Instance-Id и транзакционная идемпотентность сообщений. Сохранены публичные tests/contracts/README текущего skeleton. Служебные документы находятся отдельно в `lab4/docs/`.
|
||||
- **Схема** (`003_attachments.sql`). `attachments`: UUID PK, chat_id/owner_id FK, составной FK на членство, filename, MIME, размер 1–10 MiB, SHA-256, уникальный object_key, created_at/updated_at. `message_attachments`: FK на сообщение/вложение, position 0–9, уникальны пара и позиция. Байты не хранятся в PostgreSQL. Проверка same-chat перед INSERT связи даёт 404 для чужого/неизвестного attachment_id. Любой участник может использовать доступное вложение своего чата.
|
||||
- **Совместимость сообщения.** В ответах добавлен attachment_ids; в запросе по умолчанию []. Пустой/пробельный text допустим только с вложением. Список не содержит дублей, до 10 UUID, порядок сохраняется. Отсутствие списка и [] дают одинаковый fingerprint; пустой список не включается в fingerprint, поэтому сохраняется hash прежних text-only запросов lab3. Replay возвращает прежний Message 201, другой валидный body 409, доступ и сессия перепроверяются.
|
||||
- **Пути и ACL.** Object key — `originals/<случайный UUID>`, не filename и не публичный URL. Имя — только отображаемое поле: 1–255 Unicode code points, не пустое, без slash/backslash/NUL/control. Сначала Bearer и членство, затем загрузка; metadata/content доступны только участникам, посторонний/неизвестный объект 404. Публичный JSON не раскрывает object key, bucket, endpoint или credentials.
|
||||
- **Получение файла.** POST принимает сырой PNG/JPEG, не multipart. Content-Length проверяется заранее, но фактически полученные bytes тоже считаются. Тело читается через Request.stream в TemporaryFile на диске, SHA-256 считается по ходу. Лимит 10 MiB и абсолютное ожидание тела 30 с; слот загрузки ожидается до 1 с. Файл удаляется/закрывается в finally, постоянной локальной копии оригинала у API нет.
|
||||
- **Проверка изображения.** Pillow определяет формат и размеры до load; MIME должен совпадать. >40 млн пикселей даёт 413 до полной распаковки. verify проверяет структуру, повторный open/load — декодирование. Повреждённый/неподдерживаемый файл 415, слишком большой 413, опасное имя 400. Текстовые PNG chunks ограничены 1 MiB. На API один decoder и не более 6 одновременно обрабатываемых upload: несколько больших распаковок не размножают расход RAM. Оригинальные байты не перекодируются; crop/thumbnail/очереди относятся к следующим работам.
|
||||
- **Буферы Nginx.** На upload-route client_max_body_size 11 MiB, чтобы 10 MiB+1 дошёл до API и получил 413; client_body_buffer_size 64 KiB, остальное буферизуется на диск. `proxy_request_buffering on`, максимум 16 параллельных загрузок на LB, inactivity timeout тела 10 с, upload read/send timeout 30 с. Это сознательное дисковое буферизование, не обещание мгновенного приёма произвольно медленного клиента.
|
||||
- **201 и согласованность.** После валидации API начинает PG-транзакцию и берёт advisory lock от object_key. Затем выполняются S3 PUT и INSERT metadata; 201 возвращается только после успешного COMMIT. У S3 и PostgreSQL нет общей атомарной транзакции. Подтверждённые metadata доступны всем API, оригинал находится в общем MinIO.
|
||||
- **Компенсация неизвестного результата.** Ошибка PUT/SQL/COMMIT вызывает новую проверку под тем же lock. Объект удаляется только если БД точно подтверждает отсутствие metadata. При неизвестном COMMIT или недоступной БД слепого DeleteObject нет: уже учтённый файл должен остаться, неучтённый подберёт janitor. Обрыв неполного клиентского тела не начинает S3 PUT; аварийное завершение после PUT может оставить orphan.
|
||||
- **Автоматическая очистка.** Отдельный cleanup-service каждые 60 с листает только originals/* страницами до 1000 объектов. Grace по умолчанию 300 с, минимум 60 с. Для старого кандидата pg_try_advisory_xact_lock и свежий SELECT повторно проверяют metadata; учтённый/свежий/активно заблокированный объект пропускается. При ошибке БД удаления не происходит. Dry-run не удаляет, повтор apply идемпотентен. Здесь реализована компенсация/orphan cleanup уровня 4, не полный исследовательский reconciler из трека 5 и не восстановление missing bytes.
|
||||
- **Скачивание.** GET content возвращает StreamingBody S3 кусками 64 KiB через авторизованный API, без redirect. До 8 открытых S3-stream на API; close/release выполняется в finally ответа, включая ошибку отправки headers и disconnect. Content-Length сверяется с metadata. Variant в lab4 разрешён только original; прочие значения 400. Истинный MIME и RFC5987 filename передаются в headers.
|
||||
- **MinIO и права.** minio-init идемпотентно создаёт bucket, запрещает anonymous, создаёт отдельного API user и policy. Get/Put/Delete разрешены только originals/* этого bucket; ListBucket — только для этого prefix; GetBucketLocation — для bucket. CreateBucket, чужие bucket/prefix и admin-права не выдаются. Console слушает 127.0.0.1:9001 внутри MinIO, не опубликована и недоступна из API-сети. Приватный bucket не заменяет ACL приложения.
|
||||
- **Здоровье и таймауты.** Readiness требует PostgreSQL, совместимую схему и доступный S3 bucket с API credentials; Redis — необязательный кэш. Boto3 connect 1 с/read 3 с, одна попытка, пул/stream gate 8. PG-пул 8 на API, ожидание 2 с, statement 3 с/lock 2 с. S3 outage даёт 503 API либо 502/503/504 LB после исключения backend. Nginx OSS использует Docker DNS resolve и пассивные max_fails 1/fail_timeout 2 с; активный health_check Plus не используется. SIGTERM/rolling сохраняют схему lab3.
|
||||
|
||||
Роли, ресурсы и запуск:
|
||||
|
||||
| Service | Версия / ресурсы | Данные, сеть и готовность |
|
||||
| --- | --- | --- |
|
||||
| api×2 | `mtusi-lab4-api:lab4`; каждая 1 CPU/512 MiB, uid10001 | front+data, без host ports; HTTP ready, только ограниченные S3 secrets |
|
||||
| lb | `nginx:1.28.0-alpine`; 0.5 CPU/128 MiB | только front; единственный public endpoint 127.0.0.1:8080 |
|
||||
| postgres | `postgres:17.5-bookworm`; 1 CPU/512 MiB | internal data; postgres_data, pg_isready |
|
||||
| redis | `redis:7.4.4-bookworm`; 0.25 CPU/128 MiB | internal data; redis_data, AOF, ping |
|
||||
| minio | официальный release 2025-04-22, сборка из pinned source; 1 CPU/512 MiB, uid10002 | internal data; minio_data; HTTP /minio/health/ready; root secrets |
|
||||
| minio-init | официальный mc release 2025-04-16, сборка; 0.25 CPU/128 MiB | root+API secrets, policy read-only, Exited(0) |
|
||||
| migrate | образ API; 0.5 CPU/256 MiB | только DB secret, версии/lock/checksum, Exited(0) до API |
|
||||
| cleanup | образ API; 0.25 CPU/192 MiB | DB+ограниченные S3 secrets, только data, цикл 60 с |
|
||||
|
||||
Python 3.13.5, библиотеки и транзитивные зависимости закреплены в requirements.lock. В .env реально используется SESSION_TTL_SECONDS; S3_ENDPOINT/REGION/BUCKET и *_FILE задаются Compose. Контрактные пределы 10 MiB/40 Mp не увеличиваются через переменные. Отдельные migrate и minio-init завершаются до API. Dockerfile приложения не запускает миграции каждой репликой.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Основные эксперименты — 2026-11-02; контрольный запуск — 2026-11-09. Среда: macOS/Apple M4 Pro, 24 GiB RAM; Docker VM 12 CPU и около 7.75 GiB. Ветка `solution/bpi2303-lukyanov`. Измерения относятся к текущему учебному стенду, пароль/токен в evidence не сохранялись.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Контракт и публичный API (3) | make check/validate-container/test | PASS: lab4 public API scenario, checksum, ACL и сообщения с ids | evidence/check.txt, openapi.txt, smoke.txt |
|
||||
| Метаданные в PG, bytes в приватном S3 (3) | схема 003, security test, download | готовые originals в bucket, API не раскрывает key; anonymous 403 | migrations/003_attachments.sql, evidence/s3-security.json |
|
||||
| PNG/JPEG и исходные bytes (3) | собственные тесты + smoke | 201 ready, SHA-256/размер/bytes совпадают, image-only сообщение валидно | evidence/student-tests.txt, smoke.txt |
|
||||
| Byte/pixel/MIME/filename ограничения (3) | тесты повреждений, >10 MiB, chunked, >40 Mp header | 400/413/415; pixel limit до load, пользовательская ошибка не 500 | evidence/student-tests.txt |
|
||||
| Upload на одной, download на другой (3) | verify_attachments.py failover | остановлен точно upload-instance; другой вернул тот же hash/bytes | evidence/attachment-failover.json |
|
||||
| Полный restart с S3 volume (3) | verify_cluster.py persistence | прежняя сессия, metadata и оригинал доступны; реальные процессы перезапущены | evidence/persistence.txt, persistence.json |
|
||||
| Прежняя балансировка/идемпотентность (3) | test-cluster, собственные регрессии | concurrent replay, failover/SIGTERM/rolling проходят | evidence/concurrent-idempotency.json, public-failover.txt, sequential-failover.json, graceful-shutdown.json, rolling-update.json |
|
||||
| Все отрицательные случаи файлов/чужих ids (4) | make test-student | 21 тест PASS, включая MIME, corrupt/oversize, чужой attachment и другой чат | evidence/student-tests.txt |
|
||||
| Компенсация после PUT/DB failure (4) | SQL trigger перед metadata INSERT, затем реальный SIGKILL | SQL-сбой 503 компенсирован; orphan после kill обнаружен и удалён после grace | evidence/student-tests.txt, orphan-cleanup.json |
|
||||
| Безопасная очистка (4) | dry-run/apply/repeat; fresh/tracked/locked | dry-run: 0 удалений; apply удалил orphan; повтор 0; свежий/используемый/locked сохранены | evidence/orphan-cleanup.json |
|
||||
| Потоковая обработка и RAM при ≥10 загрузках (4) | make uploads | 10/10 одновременно, 2 API, 0 ошибок/OOM; пик 205.824/214.730 MiB при 512 MiB | evidence/concurrent-uploads.json |
|
||||
| Bucket bootstrap идемпотентен, IAM ограничен (4) | повтор minio-init; прямые S3-запросы | objects/metadata не изменились; anonymous 403, CreateBucket 403, другой prefix: 403 | evidence/s3-security.json |
|
||||
| Консоль не открыта (4) | socket из API к minio:9001; infrastructure snapshot | соединение не установлено, root secret в API не смонтирован, host ports только у LB | evidence/s3-security.json, infrastructure.json |
|
||||
| Миграции/инициализация с нуля | make test-fresh | schema 001–003 и private bucket на новых volumes, тесты PASS | evidence/fresh-migrations.txt |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Pytest также выводит предупреждение старой закреплённой botocore об utcnow в Python 3.13 и диагностическое предупреждение Pillow в тесте pixel-bomb; проверки результата 413 и все assertions проходят. Публичные проверки и контракт не ослаблялись.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Исследовательский трек 5 не выполнялся. Файлы синтетические. Обрыв тела и сбой между PUT/COMMIT проверяются раздельно: одного успешного API-запроса недостаточно для доказательства компенсации.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потери |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Upload → stop обслужившей API → download | файл читается через другую API | тот же SHA-256 и bytes, другой alias | 1.988 с от старта действия / потерь нет |
|
||||
| Обрыв HTTP upload | неполное тело не создаёт объект/metadata | из 500000 объявленных передано 108 bytes, TCP закрыт; counts не изменились | orphan нет |
|
||||
| Ошибка INSERT metadata после S3 PUT | rollback PG и компенсация S3 | HTTP 503, нет строки и нового объекта | подтверждения не было |
|
||||
| SIGKILL после PUT, до COMMIT | неподтверждённый объект может остаться и будет очищен | наблюдался exit 137; объект есть, metadata нет; после 61 с реального ожидания dry-run показал кандидата, apply удалил его | orphan устранён, 201 не выдан |
|
||||
| Repeat cleanup, fresh/linked/locked | безопасные повторы | повтор не удаляет объекты; свежий и используемый сохранены, объект под lock сохранён до unlock | потерь используемых данных нет |
|
||||
| MinIO недоступен | ограниченная ошибка без ложного 201 | upload 503 за 0.2487 с; после старта прежний оригинал читается | тот же токен, bytes сохранены |
|
||||
| Полный restart API/PG/Redis/MinIO/cleanup/Nginx | сессия и оригинал переживают restart | public persistence PASS | именованные volumes сохранены |
|
||||
|
||||
**Методика crash/cleanup:** временный BEFORE INSERT trigger задерживает ровно одно имя файла через pg_sleep(2). После наблюдения INSERT в pg_stat_activity и появления объекта в S3 API получает SIGKILL. Metadata не зафиксирована, 201 не отправлен. Контейнер восстановлен, trigger удалён. После настоящего 61-секундного ожидания применяется grace 60 с. Дополнительно проверены старый объект под advisory lock, свежий объект и старое вложение, связанное с сообщением. После снятия lock его тестовый orphan также удалён. В обычном запуске grace 300 с, цикл 60 с; при недоступной БД уборка откладывается.
|
||||
|
||||
**Память при десяти загрузках:** воспроизводимый RGB PNG 3000×1000, 9002318 bytes, zlib level 0, строки из SHAKE-256. Один неизменяемый payload используется десятью клиентскими потоками с barrier. Все десять HTTP-интервалов перекрывались 0.345 с, завершились 201, обе API наблюдались, ошибок 0. Время группы 1.563 с. Один оригинал скачан другим участником и побайтно сверен.
|
||||
|
||||
Перед опытом API пересозданы, чтобы memory.peak и VmHWM относились к этому запуску. Peak включает startup/bootstrap/загрузки/контрольное скачивание. Kernel high-water mark не пропускает короткие пики между samples:
|
||||
|
||||
| API-процесс | Cgroup memory peak | PID 1 VmHWM | Лимит | OOM |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Первый | 205.824 MiB | 182.871 MiB | 512 MiB | 0 |
|
||||
| Второй | 214.730 MiB | 183.293 MiB | 512 MiB | 0 |
|
||||
|
||||
Cgroup включает cache и вспомогательные процессы, VmHWM — resident peak API. Профиль, версии, limits, baseline/final, интервалы запросов и checksum — в evidence/concurrent-uploads.json. Опыт показывает расход памяти этого профиля, а не максимальный throughput или универсальный RSS для всех допустимых JPEG/PNG. Сценарии lab3 под текстовой нагрузкой сохранены отдельно; их числа не заменяют эксперимент с файлами.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host, одна PostgreSQL и один MinIO: отказ узла/диска и HA хранилищ не покрываются. Named volume не заменяет backup. Потерянные bytes из missing object janitor восстановить не может.
|
||||
- S3 и PostgreSQL неатомарны. Между PUT и COMMIT есть окно orphan; компенсация и grace-cleanup закрывают его после восстановления зависимостей. При недоступной БД удаление запрещено; время уборки не является SLA.
|
||||
- Janitor удаляет только неучтённые старые объекты. Учтённое, но никуда не прикреплённое вложение сохраняется: в обязательном API нет delete/retention для таких файлов.
|
||||
- Readiness считает S3 критичной зависимостью. При S3 outage Nginx может исключить обе API, и text-only запросы тоже станут недоступны, хотя PG жив. Внутренний live процесса остаётся 200.
|
||||
- Таймауты SDK/Nginx относятся к отдельным операциям I/O. После начала HTTP download нельзя заменить частичное 200 структурированным 503; клиент проверяет размер/checksum.
|
||||
- Нагрузочный замер: десять одновременных загрузок по 9 MB, две API, один прогон. Исследования максимального RPS, presigned upload, thumbnail/crop и фоновой обработки lab5 не проводились.
|
||||
- MinIO собран из закреплённого исторического релиза; стенд локальный и учебный. HTTP/Basic только на localhost с синтетическими данными; TLS, rotation, secret manager и полное разделение DB privileges оставлены следующим работам.
|
||||
- Между страницами нет общего snapshot. Сохраняются оговорённые в lab3 ограничения часов и конкурентных вставок. Breaking DB migrations потребуют отдельной стратегии.
|
||||
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
name: mtusi-lab4
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.5-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
redis:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
migrate:
|
||||
image: mtusi-lab4-api:lab4
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
api:
|
||||
image: mtusi-lab4-api:lab4
|
||||
build: .
|
||||
scale: 2
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
POOL_MAX_SIZE: '8'
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [front, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio:
|
||||
image: mtusi-lab4-minio:2025-04-22
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: minio
|
||||
secrets: [minio_root_user, minio_root_password]
|
||||
volumes: [minio_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
stop_grace_period: 10s
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio-init:
|
||||
image: mtusi-lab4-mc:2025-04-16
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: mc
|
||||
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
|
||||
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
|
||||
networks: [data]
|
||||
depends_on:
|
||||
minio: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
cleanup:
|
||||
image: mtusi-lab4-api:lab4
|
||||
command: [python, -m, messenger.cleanup, --loop, --apply, --grace-seconds, '300', --interval, '60']
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
mem_limit: 192m
|
||||
cpus: 0.25
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
|
||||
networks: [front]
|
||||
ports: ['127.0.0.1:8080:8080']
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 1s
|
||||
timeout: 8s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
minio_root_user: {file: ./secrets/minio_root_user}
|
||||
minio_root_password: {file: ./secrets/minio_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"stopped": {
|
||||
"container": "01ac5da3549e234a658a6202029fec3661951caf9cd6230f71536579465b51df",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"instance": "68057e4f671d4bc4beebe6446a39f551",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"stopped_at_utc": "2026-11-02T09:10:52.334643+00:00",
|
||||
"stop_seconds": 0.415,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"download_instance": "4bd9e299c91d4af99e1b94bab75f8f3d",
|
||||
"same_bytes": true,
|
||||
"same_sha256": true,
|
||||
"seconds_from_action_start": 1.988
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"parallel_requests": 8,
|
||||
"observed_instances": [
|
||||
"3ca208a978e441ccba13625143527d66",
|
||||
"de89643f69534e43bc1111510e0a993e"
|
||||
],
|
||||
"responses_201": 8,
|
||||
"messages_created": 1,
|
||||
"different_body_status": 409
|
||||
}
|
||||
@@ -0,0 +1,612 @@
|
||||
{
|
||||
"captured_at_utc": "2026-11-02T09:15:37.434138+00:00",
|
||||
"result": "PASS",
|
||||
"context": {
|
||||
"git_commit": "ec540a1a725862a8619fcb758daec4ad197f0dd5",
|
||||
"tracked_worktree_dirty": true,
|
||||
"host_python": "3.13.7",
|
||||
"host_os": "macOS-27.2-arm64-arm-64bit-Mach-O",
|
||||
"docker_version": "29.8.2",
|
||||
"compose_version": "5.6.0",
|
||||
"contract_version": "1.4.0",
|
||||
"endpoint": {
|
||||
"scheme": "http",
|
||||
"host": "localhost",
|
||||
"port": 8080
|
||||
}
|
||||
},
|
||||
"input": {
|
||||
"format": "PNG",
|
||||
"mode": "RGB",
|
||||
"width": 3000,
|
||||
"height": 1000,
|
||||
"size_bytes": 9002318,
|
||||
"sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe",
|
||||
"generator": "SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0",
|
||||
"same_immutable_bytes_for_all_requests": true
|
||||
},
|
||||
"concurrency": 10,
|
||||
"reset": {
|
||||
"command": "docker compose up -d --no-deps --force-recreate api",
|
||||
"purpose": "Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes"
|
||||
},
|
||||
"peak_scope": "Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps",
|
||||
"cluster_before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:15:38.382323712Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:15:38.308471337Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "cleanup",
|
||||
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
|
||||
"name": "mtusi-lab4-cleanup-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:38.357370005Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:39.330794254Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
|
||||
"name": "mtusi-lab4-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.212308463Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
|
||||
"name": "mtusi-lab4-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:12:16.586994384Z",
|
||||
"image": "mtusi-lab4-minio:2025-04-22",
|
||||
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
|
||||
"name": "mtusi-lab4-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.288107921Z",
|
||||
"image": "mtusi-lab4-mc:2025-04-16",
|
||||
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
|
||||
"name": "mtusi-lab4-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.508777334Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
|
||||
"name": "mtusi-lab4-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.445843167Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"3bd3bbc5d37c4b5a97dc188076f363dd": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 96063488,
|
||||
"cgroup_current_bytes": 87937024,
|
||||
"pid1_vmhwm_bytes": 96518144,
|
||||
"pid1_vmrss_bytes": 96518144,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
},
|
||||
"de89643f69534e43bc1111510e0a993e": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 104452096,
|
||||
"cgroup_current_bytes": 87433216,
|
||||
"pid1_vmhwm_bytes": 96370688,
|
||||
"pid1_vmrss_bytes": 96370688,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"memory_limits": [
|
||||
{
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000
|
||||
},
|
||||
{
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000
|
||||
}
|
||||
],
|
||||
"requests": [
|
||||
{
|
||||
"index": 0,
|
||||
"start_seconds": 0.9074667920067441,
|
||||
"status": 201,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"attachment_id": "14a5f6e5-0973-4b7f-98e2-81a9b7ba6ccb",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.4336796249990584
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
"start_seconds": 0.9073489169968525,
|
||||
"status": 201,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"attachment_id": "79f74d1b-624f-497e-b80d-e2605dc22427",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5235847080039093
|
||||
},
|
||||
{
|
||||
"index": 2,
|
||||
"start_seconds": 0.9075777499965625,
|
||||
"status": 201,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"attachment_id": "c5e24f93-8f8b-49fc-9e71-13161f08be4b",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.2526268750079907
|
||||
},
|
||||
{
|
||||
"index": 3,
|
||||
"start_seconds": 0.907239958003629,
|
||||
"status": 201,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"attachment_id": "480d346b-f983-45a3-9a78-05064f27ef06",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.352467999997316
|
||||
},
|
||||
{
|
||||
"index": 4,
|
||||
"start_seconds": 0.9076472919987282,
|
||||
"status": 201,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"attachment_id": "9e96aac9-21a4-4b27-b600-881e440e17b7",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5511514579993673
|
||||
},
|
||||
{
|
||||
"index": 5,
|
||||
"start_seconds": 0.9077015420043608,
|
||||
"status": 201,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"attachment_id": "85dde29d-dccf-43a8-b634-4fb380ce555f",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5152983750012936
|
||||
},
|
||||
{
|
||||
"index": 6,
|
||||
"start_seconds": 0.9077392080071149,
|
||||
"status": 201,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"attachment_id": "63ab944f-fd09-465e-bd9a-ad033cf79477",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5623937500058673
|
||||
},
|
||||
{
|
||||
"index": 7,
|
||||
"start_seconds": 0.9075430829980178,
|
||||
"status": 201,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"attachment_id": "4b5e0c15-4a6c-47a4-9741-74cd6ac3b3db",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5540448750107316
|
||||
},
|
||||
{
|
||||
"index": 8,
|
||||
"start_seconds": 0.9074074170057429,
|
||||
"status": 201,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"attachment_id": "38272fe7-3a6a-4064-9664-f7346a36431a",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5352119580056751
|
||||
},
|
||||
{
|
||||
"index": 9,
|
||||
"start_seconds": 0.9075136250030482,
|
||||
"status": 201,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"attachment_id": "5c124808-e996-4e34-bfe6-16f20fbe6895",
|
||||
"metadata_verified": true,
|
||||
"end_seconds": 1.5333833750046324
|
||||
}
|
||||
],
|
||||
"elapsed_seconds": 1.562777917002677,
|
||||
"peak_active_requests": 10,
|
||||
"all_ten_overlap_seconds": 0.3448876670008758,
|
||||
"all_ten_overlapping": true,
|
||||
"successful_count": 10,
|
||||
"error_count": 0,
|
||||
"observed_instances": [
|
||||
"3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"de89643f69534e43bc1111510e0a993e"
|
||||
],
|
||||
"representative_download": {
|
||||
"verified_as": "bob (chat member)",
|
||||
"matches_original_bytes": true,
|
||||
"size_bytes": 9002318,
|
||||
"sha256": "9bb84f9e461995a5e031a0678da59c6dea8d963a157e590a9798d9030437c3fe"
|
||||
},
|
||||
"final": {
|
||||
"3bd3bbc5d37c4b5a97dc188076f363dd": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 215822336,
|
||||
"cgroup_current_bytes": 183332864,
|
||||
"pid1_vmhwm_bytes": 191754240,
|
||||
"pid1_vmrss_bytes": 191754240,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
},
|
||||
"de89643f69534e43bc1111510e0a993e": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 225161216,
|
||||
"cgroup_current_bytes": 182919168,
|
||||
"pid1_vmhwm_bytes": 192196608,
|
||||
"pid1_vmrss_bytes": 192196608,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"cluster_after": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:15:38.382323712Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "3bd3bbc5d37c4b5a97dc188076f363dd",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:15:38.308471337Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "cleanup",
|
||||
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
|
||||
"name": "mtusi-lab4-cleanup-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:38.357370005Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:39.330794254Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
|
||||
"name": "mtusi-lab4-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.212308463Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
|
||||
"name": "mtusi-lab4-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:12:16.586994384Z",
|
||||
"image": "mtusi-lab4-minio:2025-04-22",
|
||||
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
|
||||
"name": "mtusi-lab4-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.288107921Z",
|
||||
"image": "mtusi-lab4-mc:2025-04-16",
|
||||
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
|
||||
"name": "mtusi-lab4-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.508777334Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
|
||||
"name": "mtusi-lab4-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.445843167Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"oom_killed": {
|
||||
"mtusi-lab4-api-1": false,
|
||||
"mtusi-lab4-api-2": false
|
||||
},
|
||||
"per_node_peaks": {
|
||||
"3bd3bbc5d37c4b5a97dc188076f363dd": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 215822336,
|
||||
"cgroup_current_bytes": 183332864,
|
||||
"pid1_vmhwm_bytes": 191754240,
|
||||
"pid1_vmrss_bytes": 191754240,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
},
|
||||
"de89643f69534e43bc1111510e0a993e": {
|
||||
"source": "cgroup-v2",
|
||||
"cgroup_peak_bytes": 225161216,
|
||||
"cgroup_current_bytes": 182919168,
|
||||
"pid1_vmhwm_bytes": 192196608,
|
||||
"pid1_vmrss_bytes": 192196608,
|
||||
"memory_events": {
|
||||
"low": 0,
|
||||
"high": 0,
|
||||
"max": 0,
|
||||
"oom": 0,
|
||||
"oom_kill": 0,
|
||||
"oom_group_kill": 0,
|
||||
"sock_throttled": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"redis": {
|
||||
"profile_status": 200,
|
||||
"fallback_seconds": 0.0132,
|
||||
"anonymous": 401,
|
||||
"readiness": 200
|
||||
},
|
||||
"postgres": {
|
||||
"authenticated_status": 503,
|
||||
"response_seconds": 0.0047,
|
||||
"anonymous_status": 401,
|
||||
"internal_api_liveness": [
|
||||
200,
|
||||
200
|
||||
],
|
||||
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
|
||||
},
|
||||
"same_session_after_recovery": true
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
./scripts/check_fresh_migrations.sh
|
||||
Applied 001_initial.sql
|
||||
Applied 002_idempotency.sql
|
||||
Applied 003_attachments.sql
|
||||
Migrations up to date
|
||||
Migrations up to date
|
||||
..................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 13 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
21 passed, 15 warnings in 4.23s
|
||||
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"target_instance": "69d3312a6f6b4f7bb50651c8632e196a",
|
||||
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
|
||||
"in_flight_observed_in_pg_stat_activity": true,
|
||||
"in_flight_response": 201,
|
||||
"new_tcp_after_sigterm": {
|
||||
"accepted": false,
|
||||
"error_type": "ConnectionRefusedError"
|
||||
},
|
||||
"target_still_running_when_new_tcp_rejected": true,
|
||||
"exit_code": 0,
|
||||
"shutdown_seconds_from_sigterm": 2.0927,
|
||||
"confirmed_message_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
|
||||
"name": "mtusi-lab4-api-3",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:26:04.73288638Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"release": "lab4-ce567129"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
|
||||
"name": "mtusi-lab4-api-4",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:26:10.354473716Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"release": "lab4-ce567129"
|
||||
},
|
||||
{
|
||||
"service": "cleanup",
|
||||
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
|
||||
"name": "mtusi-lab4-cleanup-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:24:53.220476166Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.272136138Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
|
||||
"name": "mtusi-lab4-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:25:13.130777342Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
|
||||
"name": "mtusi-lab4-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.261138388Z",
|
||||
"image": "mtusi-lab4-minio:2025-04-22",
|
||||
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
|
||||
"name": "mtusi-lab4-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:25:13.207299134Z",
|
||||
"image": "mtusi-lab4-mc:2025-04-16",
|
||||
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
|
||||
"name": "mtusi-lab4-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:26:21.713520555Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
|
||||
"name": "mtusi-lab4-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:26:20.609059638Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"declared_bytes": 500000,
|
||||
"sent_bytes": 108,
|
||||
"metadata_unchanged": true,
|
||||
"s3_object_count_unchanged": true
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"instance": "6e358c3478904b0e97230b8ceeb17c24",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"stopped_at_utc": "2026-11-02T09:25:37.448855+00:00",
|
||||
"stop_seconds": 0.5935,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"version": "nginx version: nginx/1.28.0",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"public_server_header": "nginx",
|
||||
"config_syntax": "PASS",
|
||||
"observed_instances": [
|
||||
"3ca208a978e441ccba13625143527d66",
|
||||
"de89643f69534e43bc1111510e0a993e"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"fault": "SIGKILL API after successful S3PUT, before metadata COMMIT",
|
||||
"crash_exit_code": 137,
|
||||
"target_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"upload_acknowledged": false,
|
||||
"orphan_confirmed": true,
|
||||
"grace_wait_seconds": 61,
|
||||
"dry_run": {
|
||||
"mode": "dry-run",
|
||||
"grace_seconds": 60,
|
||||
"listed": 29,
|
||||
"fresh": 1,
|
||||
"tracked": 26,
|
||||
"locked": 1,
|
||||
"orphans": 1,
|
||||
"deleted": 0
|
||||
},
|
||||
"apply": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 60,
|
||||
"listed": 29,
|
||||
"fresh": 1,
|
||||
"tracked": 26,
|
||||
"locked": 1,
|
||||
"orphans": 1,
|
||||
"deleted": 1
|
||||
},
|
||||
"repeat": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 60,
|
||||
"listed": 28,
|
||||
"fresh": 1,
|
||||
"tracked": 26,
|
||||
"locked": 1,
|
||||
"orphans": 0,
|
||||
"deleted": 0
|
||||
},
|
||||
"after_lock_release": {
|
||||
"mode": "apply",
|
||||
"grace_seconds": 60,
|
||||
"listed": 28,
|
||||
"fresh": 1,
|
||||
"tracked": 26,
|
||||
"locked": 0,
|
||||
"orphans": 1,
|
||||
"deleted": 1
|
||||
},
|
||||
"fresh_preserved": true,
|
||||
"tracked_and_linked_preserved": true,
|
||||
"active_lock_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:23:34.501426213Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "3ca208a978e441ccba13625143527d66",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:15:38.308471337Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "de89643f69534e43bc1111510e0a993e",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "cleanup",
|
||||
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
|
||||
"name": "mtusi-lab4-cleanup-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:38.357370005Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:39.330794254Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
|
||||
"name": "mtusi-lab4-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.212308463Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
|
||||
"name": "mtusi-lab4-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:12:16.586994384Z",
|
||||
"image": "mtusi-lab4-minio:2025-04-22",
|
||||
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
|
||||
"name": "mtusi-lab4-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:10:37.288107921Z",
|
||||
"image": "mtusi-lab4-mc:2025-04-16",
|
||||
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
|
||||
"name": "mtusi-lab4-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.508777334Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
|
||||
"name": "mtusi-lab4-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:03:32.445843167Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.617456221Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "c5dd801e4df142e599da4bafc592e9d7",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.685437846Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data",
|
||||
"mtusi-lab4_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "6e358c3478904b0e97230b8ceeb17c24",
|
||||
"release": "lab4-v1"
|
||||
},
|
||||
{
|
||||
"service": "cleanup",
|
||||
"container": "7b76cdac9466b51209de0eb07f566a25ce9829b4ac9c454b1d5d2337866908a6",
|
||||
"name": "mtusi-lab4-cleanup-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:24:53.220476166Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.272136138Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "5011a2065a877dbe6f3867961067d19ee3c8a9b4dce036a4c0bc0d0dd0fc9ce2",
|
||||
"name": "mtusi-lab4-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:24:53.965337083Z",
|
||||
"image": "mtusi-lab4-api:lab4",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "541ae9cb0aa97d512613671772597869f2b255860b8bb1ebd3360a62d160bd36",
|
||||
"name": "mtusi-lab4-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.261138388Z",
|
||||
"image": "mtusi-lab4-minio:2025-04-22",
|
||||
"image_id": "sha256:e55c472ba116fb1a1ab6dc2c94ceae51056977434e59da3e8cb3421d6442efe9",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "a354624e1306dfea39b630ea882e5ce2b6a997eaa22c85a679ee95a8da863641",
|
||||
"name": "mtusi-lab4-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T09:24:53.907159458Z",
|
||||
"image": "mtusi-lab4-mc:2025-04-16",
|
||||
"image_id": "sha256:c5b3fa838b7ae5e787473b7d95f926e587899940c6234e5487bc749ac24a23e9",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "3bbaa1721cad718b66e4e090c38ebae718cb45dc9fdcf5f241d613bfae2294b2",
|
||||
"name": "mtusi-lab4-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.284073804Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "4fd82d3c8816ce4cb86e02b81dbe6a673b309d6fa00217c5c259087b7619ac0a",
|
||||
"name": "mtusi-lab4-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.227454096Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab4_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab4_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 134, \"failed_attempts\": 0, \"first_recovery_seconds\": 1.104}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||
"stopped": {
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"instance": "c5dd801e4df142e599da4bafc592e9d7",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"stopped_at_utc": "2026-11-02T09:24:58.384960+00:00",
|
||||
"stop_seconds": 0.4473,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 134, "failed_attempts": 0, "first_recovery_seconds": 1.104}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,171 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"instance": "0e4e9b12996647ac8d03d03d79ae64fe",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961"
|
||||
},
|
||||
{
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"instance": "427b43f714ff4bc3b99698dc90cafa3b",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961"
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
|
||||
"name": "mtusi-lab4-api-3",
|
||||
"instance": "3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"release": "lab4-ce567129",
|
||||
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351"
|
||||
},
|
||||
{
|
||||
"container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
|
||||
"name": "mtusi-lab4-api-4",
|
||||
"instance": "0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"release": "lab4-ce567129",
|
||||
"image_id": "sha256:e53ef1cc7aef252b12b8f7e0136c8836a5ab075152b194a609e6c6fa1f4de351"
|
||||
}
|
||||
],
|
||||
"target_release": "lab4-ce567129",
|
||||
"lb_before": {
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.272136138Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
"lb_after": {
|
||||
"service": "lb",
|
||||
"container": "16b997261cb684aacd6c27103b59160fce9939137d83c3dcc3e42e54f9a4ae71",
|
||||
"name": "mtusi-lab4-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T09:24:50.272136138Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab4_front"
|
||||
]
|
||||
},
|
||||
"lb_not_restarted": true,
|
||||
"build_seconds": 2.7115,
|
||||
"migration_output": "Migrations up to date",
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"new_container": "102a32862f73ac850881b369827c3d0fef35d41445893bd6807200759f201e3b",
|
||||
"old_ip": "172.25.0.2",
|
||||
"new_ip": "172.25.0.5",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"0e4e9b12996647ac8d03d03d79ae64fe",
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b"
|
||||
],
|
||||
"observed_instances": [
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.6289
|
||||
},
|
||||
{
|
||||
"old_container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"new_container": "2089d486fe8dbdf85cfda30cfe2ea79de996f070343cb076dc3bceb83d18e8a0",
|
||||
"old_ip": "172.25.0.3",
|
||||
"new_ip": "172.25.0.2",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b"
|
||||
],
|
||||
"observed_instances": [
|
||||
"0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.4987
|
||||
}
|
||||
],
|
||||
"during_replacement": {
|
||||
"seconds": 11.1277,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8499,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 763.77,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.34,
|
||||
"p95": 7.505,
|
||||
"p99": 37.719
|
||||
},
|
||||
"instances": [
|
||||
"0e4e9b12996647ac8d03d03d79ae64fe",
|
||||
"0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b"
|
||||
]
|
||||
},
|
||||
"total": {
|
||||
"seconds": 20.7958,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 15875,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 763.377,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.479,
|
||||
"p95": 7.406,
|
||||
"p99": 28.635
|
||||
},
|
||||
"instances": [
|
||||
"0e4e9b12996647ac8d03d03d79ae64fe",
|
||||
"0f0ee5b42e754b7f9aa359e746294e8f",
|
||||
"3715a1c3cd074e9f952ad3b9bb646d54",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 15875,
|
||||
"history_messages": 15875,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"upload_status_during_outage": 503,
|
||||
"response_seconds": 0.2487,
|
||||
"original_preserved_after_restart": true,
|
||||
"same_session_after_recovery": true
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"anonymous_get": 403,
|
||||
"create_bucket_with_api_credentials": 403,
|
||||
"list_other_prefix": 403,
|
||||
"console_reachable_from_api": false,
|
||||
"root_secret_mounted_in_api": false,
|
||||
"bootstrap_repeated_without_data_loss": true,
|
||||
"bootstrap_output": "PASS: private bucket and restricted API policy ready (credentials hidden)."
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"baseline": {
|
||||
"seconds": 3.0101,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 2029,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 674.073,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.838,
|
||||
"p95": 7.446,
|
||||
"p99": 24.831
|
||||
},
|
||||
"instances": [
|
||||
"118d370a438c4c14aabe117988bc86e9",
|
||||
"6e358c3478904b0e97230b8ceeb17c24"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"stopped": {
|
||||
"container": "3160c34884b51370acb5692d6646ec690408b54fbe406b82d8069146a5ca305c",
|
||||
"name": "mtusi-lab4-api-1",
|
||||
"instance": "118d370a438c4c14aabe117988bc86e9",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"stopped_at_utc": "2026-11-02T09:25:23.676868+00:00",
|
||||
"stop_seconds": 0.5348,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0006,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 3857,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 482.09,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.513,
|
||||
"p95": 16.08,
|
||||
"p99": 56.428
|
||||
},
|
||||
"instances": [
|
||||
"118d370a438c4c14aabe117988bc86e9",
|
||||
"6e358c3478904b0e97230b8ceeb17c24"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.2915,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0308,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.2842,
|
||||
"seconds_from_stop_completion": 0.0235
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
},
|
||||
{
|
||||
"stopped": {
|
||||
"container": "e4079b051482cd9397de303e7afce2b627d01205688e0041734f136596252f72",
|
||||
"name": "mtusi-lab4-api-2",
|
||||
"instance": "6e358c3478904b0e97230b8ceeb17c24",
|
||||
"release": "lab4-v1",
|
||||
"image_id": "sha256:c85561107f5a367fdfe410d70321a94dcf9ceaf305ca5b73c00525759a1c5961",
|
||||
"stopped_at_utc": "2026-11-02T09:25:37.448855+00:00",
|
||||
"stop_seconds": 0.5935,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0018,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 4137,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 517.011,
|
||||
"error_ratio": 0.000242,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.55,
|
||||
"p95": 14.11,
|
||||
"p99": 53.684
|
||||
},
|
||||
"instances": [
|
||||
"69d3312a6f6b4f7bb50651c8632e196a",
|
||||
"6e358c3478904b0e97230b8ceeb17c24"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.2502,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0189,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.2386,
|
||||
"seconds_from_stop_completion": 0.0073
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 30.9392,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17688,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 571.702,
|
||||
"error_ratio": 5.7e-05,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.77,
|
||||
"p95": 12.069,
|
||||
"p99": 46.873
|
||||
},
|
||||
"instances": [
|
||||
"118d370a438c4c14aabe117988bc86e9",
|
||||
"427b43f714ff4bc3b99698dc90cafa3b",
|
||||
"69d3312a6f6b4f7bb50651c8632e196a",
|
||||
"6e358c3478904b0e97230b8ceeb17c24"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 17688,
|
||||
"history_messages": 17688,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"each_original_container_tested": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,14 @@
|
||||
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
..................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 13 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
21 passed, 15 warnings in 4.15s
|
||||
@@ -0,0 +1,28 @@
|
||||
# Official public images/binaries were removed upstream; build fixed release sources.
|
||||
FROM golang:1.24.2-bookworm AS minio-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
|
||||
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
|
||||
|
||||
FROM golang:1.24.2-bookworm AS mc-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
|
||||
|
||||
FROM python:3.13.5-slim AS minio
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
|
||||
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY infra/minio/server.sh /server.sh
|
||||
USER 10002
|
||||
ENTRYPOINT ["/bin/sh", "/server.sh"]
|
||||
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
|
||||
|
||||
FROM debian:bookworm-slim AS mc
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect":"Allow","Action":["s3:GetBucketLocation"],"Resource":["arn:aws:s3:::messenger-attachments"]},
|
||||
{"Effect":"Allow","Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::messenger-attachments"],"Condition":{"StringLike":{"s3:prefix":["originals/*"]}}},
|
||||
{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject"],"Resource":["arn:aws:s3:::messenger-attachments/originals/*"]}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments >/dev/null
|
||||
mc anonymous set none local/messenger-attachments >/dev/null
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
|
||||
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
|
||||
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,62 @@
|
||||
worker_processes 1;
|
||||
error_log /dev/stderr warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
server_tokens off;
|
||||
limit_conn_zone $server_name zone=uploads:64k;
|
||||
|
||||
# Docker DNS; resolve updates the upstream when replicas get new IPs.
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
# All A records of api participate in the default round-robin balancing.
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name messenger;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
limit_conn uploads 16;
|
||||
client_max_body_size 11m;
|
||||
client_body_buffer_size 64k;
|
||||
client_body_timeout 10s;
|
||||
proxy_request_buffering on;
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
|
||||
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
|
||||
# must not be resent by Nginx after an ambiguous response.
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_history_idx ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Additive: old lab2 tables and messages remain valid.
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
|
||||
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position)
|
||||
);
|
||||
@@ -0,0 +1,51 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.9.0
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
attrs==26.1.0
|
||||
boto3==1.38.27
|
||||
botocore==1.38.46
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httpx==0.28.1
|
||||
idna==3.20
|
||||
iniconfig==2.3.1
|
||||
jmespath==1.1.0
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.13.0
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pillow==11.3.0
|
||||
pluggy==1.7.0
|
||||
psycopg==3.2.9
|
||||
psycopg-binary==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
pycparser==3.12
|
||||
pydantic==2.11.5
|
||||
pydantic_core==2.33.2
|
||||
Pygments==2.21.0
|
||||
pytest==8.4.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyYAML==6.0.3
|
||||
redis==6.2.0
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
s3transfer==0.13.1
|
||||
six==1.17.0
|
||||
sniffio==1.3.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.3
|
||||
@@ -0,0 +1,12 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.3
|
||||
psycopg[binary]==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.2.0
|
||||
argon2-cffi==25.1.0
|
||||
pytest==8.4.0
|
||||
httpx==0.28.1
|
||||
pydantic==2.11.5
|
||||
anyio==4.9.0
|
||||
boto3==1.38.27
|
||||
Pillow==11.3.0
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Container-side privileged test utilities. Input over stdin; never output credentials."""
|
||||
import base64
|
||||
from datetime import datetime,timezone
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore import UNSIGNED
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import ClientError
|
||||
import psycopg
|
||||
from psycopg import sql
|
||||
from psycopg.rows import dict_row
|
||||
from messenger.db import setting,Database
|
||||
from messenger.object_store import ObjectStore
|
||||
from messenger.cleanup import sweep
|
||||
|
||||
|
||||
def main():
|
||||
p=json.load(sys.stdin)
|
||||
action=p['action']
|
||||
store=ObjectStore()
|
||||
result={}
|
||||
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as conn:
|
||||
if action=='lookup':
|
||||
result=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(p['id'],)).fetchone()
|
||||
elif action=='counts':
|
||||
result={'attachments':conn.execute('SELECT count(*) AS n FROM attachments').fetchone()['n'],
|
||||
'objects':sum(len(page.get('Contents',[])) for page in store.pages())}
|
||||
elif action=='list':
|
||||
result={'keys':[obj['Key'] for page in store.pages() for obj in page.get('Contents',[])]}
|
||||
elif action=='put':
|
||||
raw=base64.b64decode(p['bytes'])
|
||||
store.put(p['key'],io.BytesIO(raw),len(raw),'image/png')
|
||||
result={'created':True}
|
||||
elif action=='delete':
|
||||
store.delete(p['key'])
|
||||
result={'deleted':True}
|
||||
elif action=='exists':
|
||||
try:
|
||||
obj=store.client.head_object(Bucket=store.bucket,Key=p['key'])
|
||||
result={'exists':True,'length':obj['ContentLength']}
|
||||
except ClientError as exc:
|
||||
if exc.response['ResponseMetadata']['HTTPStatusCode']!=404:
|
||||
raise
|
||||
result={'exists':False}
|
||||
elif action=='tracked':
|
||||
result={'tracked':conn.execute('SELECT count(*) AS n FROM attachments WHERE object_key=%s',(p['key'],)).fetchone()['n']}
|
||||
elif action=='security':
|
||||
unsigned=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT'),region_name='us-east-1',config=Config(signature_version=UNSIGNED,s3={'addressing_style':'path'},connect_timeout=1,read_timeout=3,retries={'total_max_attempts':1}))
|
||||
try:
|
||||
unsigned.get_object(Bucket=store.bucket,Key=p['key'])
|
||||
anonymous=200
|
||||
except ClientError as exc:
|
||||
anonymous=exc.response['ResponseMetadata']['HTTPStatusCode']
|
||||
try:
|
||||
store.client.create_bucket(Bucket='forbidden-'+uuid.uuid4().hex)
|
||||
create_bucket=200
|
||||
except ClientError as exc:
|
||||
create_bucket=exc.response['ResponseMetadata']['HTTPStatusCode']
|
||||
try:
|
||||
store.client.list_objects_v2(Bucket=store.bucket,Prefix='forbidden/')
|
||||
other_prefix=200
|
||||
except ClientError as exc:
|
||||
other_prefix=exc.response['ResponseMetadata']['HTTPStatusCode']
|
||||
try:
|
||||
sock=socket.create_connection(('minio',9001),timeout=1)
|
||||
sock.close()
|
||||
console=True
|
||||
except OSError:
|
||||
console=False
|
||||
result={'anonymous_get':anonymous,'create_bucket_with_api_credentials':create_bucket,
|
||||
'list_other_prefix':other_prefix,'console_reachable_from_api':console,
|
||||
'root_secret_mounted_in_api':Path('/run/secrets/minio_root_password').exists()}
|
||||
unsigned.close()
|
||||
elif action=='install':
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['filename'])))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif action=='remove':
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(p['trigger'])))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
elif action=='waiting':
|
||||
result={'waiting':conn.execute("SELECT count(*) AS n FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO attachments%%'").fetchone()['n']}
|
||||
elif action=='sweep':
|
||||
db=Database()
|
||||
db.open()
|
||||
try:
|
||||
result=sweep(db,store,grace_seconds=p.get('grace',60),apply=p.get('apply',False))
|
||||
finally:
|
||||
db.close()
|
||||
else:
|
||||
raise ValueError('Unknown check action')
|
||||
store.close()
|
||||
print(json.dumps(result,default=str))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path('secrets')
|
||||
p = root / 'postgres_password'
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24))
|
||||
pw = p.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
'minio_root_user': 'lab4-admin',
|
||||
'minio_root_password': secrets.token_hex(24),
|
||||
's3_access_key_id': 'messenger-api',
|
||||
's3_secret_access_key': secrets.token_hex(24),
|
||||
}.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Bind-mounted Docker secrets must be readable by container uid 10001.
|
||||
# Parent directory remains 0700 on the host.
|
||||
for path in root.iterdir():
|
||||
path.chmod(0o444)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready (values hidden).')
|
||||
PY
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture repeatable short load run and sanitized environment/resource evidence."""
|
||||
import datetime
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import cluster
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def output(args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def counts():
|
||||
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
|
||||
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
|
||||
|
||||
|
||||
def resource_summary(samples):
|
||||
peaks = {}
|
||||
for sample in samples:
|
||||
for container in sample['containers']:
|
||||
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
|
||||
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
|
||||
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
|
||||
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
|
||||
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
|
||||
return {'samples':len(samples),'per_container_peaks':peaks}
|
||||
|
||||
|
||||
def main():
|
||||
names = cluster.container_ids()
|
||||
environment = {
|
||||
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'host_os': platform.platform(), 'host_architecture': platform.machine(),
|
||||
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
|
||||
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
|
||||
'dataset_before': counts(),
|
||||
'limits': {'api_each':'1 CPU / 512 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
|
||||
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
|
||||
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
|
||||
}
|
||||
if sys.platform == 'darwin':
|
||||
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
|
||||
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
|
||||
samples = []
|
||||
with (ROOT/'evidence/load-output.txt').open('w') as log:
|
||||
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
|
||||
while proc.poll() is None:
|
||||
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
|
||||
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
|
||||
for row in rows.splitlines()]})
|
||||
time.sleep(1)
|
||||
if proc.returncode:
|
||||
raise SystemExit(proc.returncode)
|
||||
environment['dataset_after'] = counts()
|
||||
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
|
||||
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
|
||||
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
|
||||
print((ROOT/'evidence/load.json').read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,214 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ten overlapping PNG uploads with kernel-recorded memory peaks; stdlib only.
|
||||
|
||||
Recreates API containers (never volumes) to reset memory high-water marks.
|
||||
Run from any directory: python3 scripts/capture_uploads.py.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
WORKERS = 10
|
||||
WIDTH, HEIGHT = 3000, 1000
|
||||
EVIDENCE = ROOT / 'evidence/concurrent-uploads.json'
|
||||
|
||||
|
||||
def make_png():
|
||||
"""Stable, incompressible RGB rows without a host image library."""
|
||||
def chunk(kind, data):
|
||||
return (struct.pack('!I', len(data)) + kind + data
|
||||
+ struct.pack('!I', zlib.crc32(kind + data) & 0xffffffff))
|
||||
|
||||
compressor = zlib.compressobj(level=0)
|
||||
compressed = []
|
||||
for row in range(HEIGHT):
|
||||
pixels = hashlib.shake_256(b'lab4-concurrent-upload-v1:' + str(row).encode()).digest(WIDTH * 3)
|
||||
compressed.append(compressor.compress(b'\x00' + pixels))
|
||||
compressed.append(compressor.flush())
|
||||
return (b'\x89PNG\r\n\x1a\n'
|
||||
+ chunk(b'IHDR', struct.pack('!IIBBBBB', WIDTH, HEIGHT, 8, 2, 0, 0, 0))
|
||||
+ chunk(b'IDAT', b''.join(compressed)) + chunk(b'IEND', b''))
|
||||
|
||||
|
||||
def memory(container):
|
||||
# cgroup peaks include allocations between observations; VmHWM is PID 1's
|
||||
# resident peak, whereas cgroup memory also includes file cache and children.
|
||||
code = '''import json,pathlib
|
||||
def number(path):
|
||||
return int(pathlib.Path(path).read_text().strip())
|
||||
root=pathlib.Path('/sys/fs/cgroup')
|
||||
if (root/'memory.peak').exists():
|
||||
source='cgroup-v2'
|
||||
peak=number(root/'memory.peak'); current=number(root/'memory.current')
|
||||
events=dict((k,int(v)) for k,v in (line.split() for line in (root/'memory.events').read_text().splitlines()))
|
||||
else:
|
||||
root=root/'memory'; source='cgroup-v1'
|
||||
peak=number(root/'memory.max_usage_in_bytes'); current=number(root/'memory.usage_in_bytes')
|
||||
events={'failcnt':number(root/'memory.failcnt')}
|
||||
status=dict(line.split(':',1) for line in pathlib.Path('/proc/1/status').read_text().splitlines() if ':' in line)
|
||||
print(json.dumps({'source':source,'cgroup_peak_bytes':peak,'cgroup_current_bytes':current,
|
||||
'pid1_vmhwm_bytes':int(status['VmHWM'].split()[0])*1024,
|
||||
'pid1_vmrss_bytes':int(status['VmRSS'].split()[0])*1024,'memory_events':events}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker', 'exec', container, 'python', '-c', code]))
|
||||
|
||||
|
||||
def version_context():
|
||||
revision = cluster.command(['git', 'rev-parse', 'HEAD'])
|
||||
dirty = subprocess.run(['git', 'diff', '--quiet'], cwd=ROOT, check=False).returncode != 0
|
||||
address = urllib.parse.urlsplit(cluster.BASE_URL)
|
||||
return {
|
||||
'git_commit': revision, 'tracked_worktree_dirty': dirty,
|
||||
'host_python': platform.python_version(), 'host_os': platform.platform(),
|
||||
'docker_version': cluster.command(['docker', 'version', '--format', '{{.Server.Version}}']),
|
||||
'compose_version': cluster.compose('version', '--short'),
|
||||
'contract_version': json.loads((ROOT / 'lab.json').read_text())['contract_version'],
|
||||
'endpoint': {'scheme': address.scheme, 'host': address.hostname, 'port': address.port},
|
||||
}
|
||||
|
||||
|
||||
def experiment(evidence):
|
||||
payload = make_png()
|
||||
check(9_000_000 <= len(payload) <= 10 * 1024 * 1024, 'generated PNG size outside experiment bounds')
|
||||
digest = hashlib.sha256(payload).hexdigest()
|
||||
evidence.update({
|
||||
'context': version_context(),
|
||||
'input': {'format': 'PNG', 'mode': 'RGB', 'width': WIDTH, 'height': HEIGHT,
|
||||
'size_bytes': len(payload), 'sha256': digest,
|
||||
'generator': 'SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0',
|
||||
'same_immutable_bytes_for_all_requests': True},
|
||||
'concurrency': WORKERS,
|
||||
'reset': {'command': 'docker compose up -d --no-deps --force-recreate api',
|
||||
'purpose': 'Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes'},
|
||||
'peak_scope': 'Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps',
|
||||
})
|
||||
cluster.compose('up', '-d', '--no-deps', '--force-recreate', 'api')
|
||||
containers = cluster.container_ids('api')
|
||||
check(len(containers) == 2, 'experiment requires exactly two API containers')
|
||||
for container in containers:
|
||||
cluster.wait_healthy(container)
|
||||
cluster.wait_cluster(2)
|
||||
replicas = cluster.replicas()
|
||||
evidence['cluster_before'] = cluster.snapshot()
|
||||
evidence['baseline'] = {item['instance']: memory(item['container']) for item in replicas}
|
||||
limits = [item for item in evidence['cluster_before'] if item['service'] == 'api']
|
||||
evidence['memory_limits'] = [
|
||||
{key: item[key] for key in ('instance', 'memory_bytes', 'nano_cpus')} for item in limits]
|
||||
check(all(item['memory_bytes'] == 512 * 1024 * 1024 and item['nano_cpus'] == 1_000_000_000
|
||||
for item in limits), 'API limits must be 512 MiB and 1 CPU each')
|
||||
client = Client(timeout=180)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
barrier = threading.Barrier(WORKERS, timeout=30)
|
||||
lock = threading.Lock()
|
||||
activity = {'active': 0, 'peak': 0}
|
||||
origin = time.monotonic()
|
||||
|
||||
def upload(index):
|
||||
worker = Client(timeout=180)
|
||||
barrier.wait()
|
||||
with lock:
|
||||
start = time.monotonic() - origin
|
||||
activity['active'] += 1
|
||||
activity['peak'] = max(activity['peak'], activity['active'])
|
||||
item = {'index': index, 'start_seconds': start}
|
||||
attachment = None
|
||||
try:
|
||||
response = worker.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent-%02d.png' % index,
|
||||
expected=201, actor=alice, raw=payload,
|
||||
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
attachment = response['json']
|
||||
item.update({'status': response['status'], 'instance': response['headers'].get('X-Instance-Id')})
|
||||
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(payload),
|
||||
'uploaded original checksum/size mismatch')
|
||||
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == alice['id'],
|
||||
'uploaded attachment ownership mismatch')
|
||||
check(attachment['status'] == 'ready', 'uploaded attachment not ready')
|
||||
check(attachment['content_type'] == 'image/png', 'uploaded MIME type mismatch')
|
||||
item.update({'attachment_id': attachment['id'], 'metadata_verified': True})
|
||||
except Failure as exc:
|
||||
item['error'] = str(exc)
|
||||
except Exception as exc:
|
||||
# Never serialize arbitrary library exception text, request or actor.
|
||||
item['error'] = type(exc).__name__
|
||||
finally:
|
||||
with lock:
|
||||
item['end_seconds'] = time.monotonic() - origin
|
||||
activity['active'] -= 1
|
||||
return item, attachment
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
||||
completed = list(pool.map(upload, range(WORKERS)))
|
||||
requests = [item for item, _attachment in completed]
|
||||
evidence['requests'] = requests
|
||||
evidence['elapsed_seconds'] = time.monotonic() - origin
|
||||
evidence['peak_active_requests'] = activity['peak']
|
||||
overlap = min(item['end_seconds'] for item in requests) - max(item['start_seconds'] for item in requests)
|
||||
evidence['all_ten_overlap_seconds'] = max(0, overlap)
|
||||
evidence['all_ten_overlapping'] = overlap > 0
|
||||
evidence['successful_count'] = sum('error' not in item for item in requests)
|
||||
evidence['error_count'] = WORKERS - evidence['successful_count']
|
||||
evidence['observed_instances'] = sorted({item['instance'] for item in requests if item.get('instance')})
|
||||
successful = [attachment for item, attachment in completed if 'error' not in item]
|
||||
if successful:
|
||||
download = client.call('GET', '/api/v1/attachments/' + successful[0]['id'] + '/content', actor=bob)
|
||||
check(download['bytes'] == payload, 'representative download differs from original')
|
||||
evidence['representative_download'] = {
|
||||
'verified_as': 'bob (chat member)', 'matches_original_bytes': True,
|
||||
'size_bytes': len(download['bytes']), 'sha256': hashlib.sha256(download['bytes']).hexdigest()}
|
||||
evidence['final'] = {item['instance']: memory(item['container']) for item in replicas}
|
||||
evidence['cluster_after'] = cluster.snapshot()
|
||||
states = cluster.inspect(containers)
|
||||
evidence['oom_killed'] = {item['Name'].lstrip('/'): item['State']['OOMKilled'] for item in states}
|
||||
evidence['per_node_peaks'] = evidence['final']
|
||||
check(evidence['successful_count'] == WORKERS, 'not all ten uploads succeeded')
|
||||
check(len(evidence['observed_instances']) >= 2, 'uploads did not reach both API instances')
|
||||
check(overlap > 0 and activity['peak'] == WORKERS, 'ten requests did not all overlap')
|
||||
check(not any(evidence['oom_killed'].values()), 'an API container was OOM-killed')
|
||||
for item in replicas:
|
||||
before, after = evidence['baseline'][item['instance']], evidence['final'][item['instance']]
|
||||
check(after['cgroup_peak_bytes'] <= 512 * 1024 * 1024, 'API exceeded configured memory limit')
|
||||
for event in ('oom', 'oom_kill', 'failcnt'):
|
||||
check(before['memory_events'].get(event, 0) == 0 and after['memory_events'].get(event, 0) == 0,
|
||||
'API recorded an OOM or cgroup allocation failure')
|
||||
check(all(item['State']['Running'] and item['State'].get('Health', {}).get('Status') == 'healthy'
|
||||
for item in states), 'both APIs must remain running and healthy')
|
||||
cluster.wait_cluster(2)
|
||||
|
||||
|
||||
def main():
|
||||
evidence = {'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'result': 'FAIL'}
|
||||
try:
|
||||
experiment(evidence)
|
||||
evidence['result'] = 'PASS'
|
||||
except Failure as exc:
|
||||
evidence['failure'] = str(exc)
|
||||
except Exception as exc:
|
||||
evidence['failure'] = type(exc).__name__
|
||||
finally:
|
||||
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
|
||||
EVIDENCE.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
|
||||
print(evidence['result'] + ': evidence/concurrent-uploads.json')
|
||||
if evidence['result'] != 'PASS':
|
||||
print(evidence.get('failure', 'experiment failed'))
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# A unique disposable Compose project; never touches normal project volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
project="mtusi-lab4-migration-check-$$"
|
||||
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
|
||||
echo 'Refusing to reuse a pre-existing scratch project' >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup() {
|
||||
docker compose -p "$project" down --volumes >/dev/null 2>&1
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
|
||||
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
|
||||
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','minio','cleanup','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/cluster.py restart
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec python3 scripts/cluster.py stop-one
|
||||
@@ -0,0 +1,250 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 attachment evidence with real S3 faults, process crash and grace cleanup."""
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,Failure,check
|
||||
|
||||
|
||||
def save(name,value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:',name,flush=True)
|
||||
|
||||
|
||||
def internal(data,container=None):
|
||||
container=container or cluster.container_ids('api')[0]
|
||||
p=subprocess.run(['docker','exec','-i',container,'python','scripts/attachment_checks.py'],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=30)
|
||||
return json.loads(p.stdout)
|
||||
|
||||
|
||||
def upload(c,actor,chat,raw,filename='sample.png'):
|
||||
return c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename='+urllib.parse.quote(filename),
|
||||
201,actor=actor,raw=raw,headers={'Content-Type':'image/png'},schema='Attachment')
|
||||
|
||||
|
||||
def private_bucket():
|
||||
c=Client()
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
response=upload(c,alice,chat,raw)
|
||||
att=response['json']
|
||||
key=internal({'action':'lookup','id':att['id']})['object_key']
|
||||
result=internal({'action':'security','key':key})
|
||||
check(result['anonymous_get']==403,'private object is accessible without S3 authentication')
|
||||
check(result['create_bucket_with_api_credentials']==403 and result['list_other_prefix']==403,'API S3 credentials are too broad')
|
||||
check(not result['console_reachable_from_api'] and not result['root_secret_mounted_in_api'],'root/console exposed to API')
|
||||
before=internal({'action':'counts'})
|
||||
output=cluster.compose('run','--rm','--no-deps','minio-init')
|
||||
check(internal({'action':'counts'})==before,'repeated bucket bootstrap changed data')
|
||||
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
check(download['bytes']==raw,'bootstrap changed original bytes')
|
||||
save('s3-security.json',{'result':'PASS',**result,'bootstrap_repeated_without_data_loss':True,'bootstrap_output':output})
|
||||
|
||||
|
||||
def upload_failover():
|
||||
cluster.wait_cluster()
|
||||
c=Client()
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
response=upload(c,alice,chat,raw)
|
||||
att=response['json']
|
||||
alias=response['headers']['X-Instance-Id']
|
||||
c.message(alice,chat,'',attachments=[att['id']])
|
||||
target=next(node for node in cluster.replicas() if node['instance']==alias)
|
||||
started=time.monotonic()
|
||||
try:
|
||||
stopped=cluster.stop_one(alias)
|
||||
deadline=started+10
|
||||
while True:
|
||||
try:
|
||||
download=c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<deadline,'file download did not recover in10s')
|
||||
time.sleep(.1)
|
||||
check(download['headers']['X-Instance-Id']!=alias,'download came from stopped process')
|
||||
check(download['bytes']==raw and hashlib.sha256(download['bytes']).hexdigest()==att['sha256'],'file changed across replicas')
|
||||
check(not cluster.inspect([target['container']])[0]['State']['Running'],'target was not left stopped')
|
||||
result={'result':'PASS','stopped':stopped,'download_instance':download['headers']['X-Instance-Id'],
|
||||
'same_bytes':True,'same_sha256':True,'seconds_from_action_start':round(time.monotonic()-started,4)}
|
||||
finally:
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('attachment-failover.json',result)
|
||||
|
||||
|
||||
def interrupted_upload():
|
||||
c=Client()
|
||||
alice,_,_,chat=c.bootstrap()
|
||||
before=internal({'action':'counts'})
|
||||
parsed=urllib.parse.urlsplit(c.base)
|
||||
sock=socket.create_connection((parsed.hostname,parsed.port or 80),timeout=3)
|
||||
path='/api/v1/chats/'+chat['id']+'/attachments?filename=interrupted.png'
|
||||
headers=('POST '+path+' HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer '+alice['token']+
|
||||
'\r\nContent-Type: image/png\r\nContent-Length: 500000\r\nConnection: close\r\n\r\n')
|
||||
sock.sendall(headers.encode()+b'\x89PNG\r\n\x1a\n'+b'x'*100)
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
sock.close()
|
||||
time.sleep(.5)
|
||||
after=internal({'action':'counts'})
|
||||
check(before==after,'interrupted request left an object or metadata')
|
||||
save('interrupted-upload.json',{'result':'PASS','declared_bytes':500000,'sent_bytes':108,
|
||||
'metadata_unchanged':True,'s3_object_count_unchanged':True})
|
||||
|
||||
|
||||
def crash_orphan():
|
||||
cluster.wait_cluster()
|
||||
c=Client()
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
nodes=cluster.replicas()
|
||||
target,survivor=nodes[0],nodes[1]
|
||||
filename='crash_'+uuid.uuid4().hex+'.png'
|
||||
fixture={'filename':filename,'function':'crash_'+uuid.uuid4().hex,'trigger':'crash_'+uuid.uuid4().hex}
|
||||
before=set(internal({'action':'list'},survivor['container'])['keys'])
|
||||
internal({'action':'install',**fixture},survivor['container'])
|
||||
code='''import json,base64,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=base64.b64decode(p['bytes']),method='POST',headers={'Authorization':'Bearer '+p['token'],'Content-Type':'image/png'})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=8)
|
||||
print(json.dumps({'status':response.code}))
|
||||
except Exception as exc:
|
||||
print(json.dumps({'status':None,'error_type':type(exc).__name__}))
|
||||
'''
|
||||
data={'address':cluster.front_address(target['container']),'path':'/api/v1/chats/'+chat['id']+'/attachments?filename='+filename,
|
||||
'bytes':base64.b64encode(raw).decode(),'token':alice['token']}
|
||||
def post():
|
||||
p=subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,input=json.dumps(data),capture_output=True,text=True,check=True,timeout=12)
|
||||
return json.loads(p.stdout)
|
||||
pool=ThreadPoolExecutor(max_workers=1)
|
||||
future=pool.submit(post)
|
||||
try:
|
||||
deadline=time.monotonic()+5
|
||||
while not internal({'action':'waiting'},survivor['container'])['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe PUT followed by pending metadata INSERT')
|
||||
time.sleep(.05)
|
||||
after=set(internal({'action':'list'},survivor['container'])['keys'])
|
||||
created=after-before
|
||||
check(len(created)==1,'expected exactly one new S3 object during pending metadata INSERT')
|
||||
orphan=created.pop()
|
||||
cluster.command(['docker','kill','--signal','SIGKILL',target['container']])
|
||||
crashed=cluster.inspect([target['container']])[0]['State']
|
||||
check(not crashed['Running'] and crashed['ExitCode']==137,'SIGKILL did not produce the expected stopped state')
|
||||
observed_exit_code=crashed['ExitCode']
|
||||
response=future.result(timeout=12)
|
||||
check(response['status']!=201,'crashed request returned an acknowledgement')
|
||||
internal({'action':'remove',**fixture},survivor['container'])
|
||||
check(internal({'action':'tracked','key':orphan},survivor['container'])['tracked']==0,'metadata unexpectedly committed after crash')
|
||||
check(internal({'action':'exists','key':orphan},survivor['container'])['exists'],'crash did not leave an orphan to clean')
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
internal({'action':'remove',**fixture},survivor['container'])
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
# A real orphan from crash, plus an old locked object and a tracked/linked attachment.
|
||||
locked='originals/'+uuid.uuid4().hex
|
||||
internal({'action':'put','key':locked,'bytes':base64.b64encode(raw).decode()})
|
||||
tracked=upload(c,alice,chat,raw)['json']
|
||||
c.message(alice,chat,'',attachments=[tracked['id']])
|
||||
print('Waiting61s for the enforced60s cleanup grace (no time bypass).',flush=True)
|
||||
time.sleep(61)
|
||||
fresh='originals/'+uuid.uuid4().hex
|
||||
internal({'action':'put','key':fresh,'bytes':base64.b64encode(raw).decode()})
|
||||
lock_code='''import json,sys,psycopg
|
||||
from messenger.db import setting
|
||||
key=sys.stdin.readline().strip()
|
||||
with psycopg.connect(setting('DATABASE_URL')) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
|
||||
print('locked',flush=True)
|
||||
sys.stdin.readline()
|
||||
'''
|
||||
holder=subprocess.Popen(['docker','exec','-i',survivor['container'],'python','-u','-c',lock_code],cwd=ROOT,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
||||
try:
|
||||
holder.stdin.write(locked+'\n')
|
||||
holder.stdin.flush()
|
||||
check(holder.stdout.readline().strip()=='locked','failed to hold cleanup coordination lock')
|
||||
dry=internal({'action':'sweep','grace':60,'apply':False})
|
||||
check(dry['deleted']==0 and dry['orphans']>=1 and dry['locked']>=1,'dry-run/grace/lock selection failed')
|
||||
check(internal({'action':'exists','key':orphan})['exists'],'dry-run deleted an object')
|
||||
applied=internal({'action':'sweep','grace':60,'apply':True})
|
||||
check(applied['deleted']>=1,'orphan not removed after grace')
|
||||
check(not internal({'action':'exists','key':orphan})['exists'],'crash orphan remained after cleanup')
|
||||
check(internal({'action':'exists','key':locked})['exists'],'active upload lock was ignored')
|
||||
check(internal({'action':'exists','key':fresh})['exists'],'fresh object was deleted')
|
||||
repeat=internal({'action':'sweep','grace':60,'apply':True})
|
||||
check(repeat['deleted']==0,'repeated sweep was not idempotent')
|
||||
check(c.call('GET','/api/v1/attachments/'+tracked['id']+'/content',actor=bob)['bytes']==raw,'tracked/linked file was deleted')
|
||||
finally:
|
||||
holder.stdin.write('\n')
|
||||
holder.stdin.flush()
|
||||
holder.stdin.close()
|
||||
holder.wait(timeout=5)
|
||||
released=internal({'action':'sweep','grace':60,'apply':True})
|
||||
check(not internal({'action':'exists','key':locked})['exists'],'unlocked old orphan not removed')
|
||||
internal({'action':'delete','key':fresh})
|
||||
save('orphan-cleanup.json',{'result':'PASS','fault':'SIGKILL API after successful S3PUT, before metadata COMMIT',
|
||||
'crash_exit_code':observed_exit_code,'target_container':target['container'],'upload_acknowledged':False,'orphan_confirmed':True,'grace_wait_seconds':61,
|
||||
'dry_run':dry,'apply':applied,'repeat':repeat,'after_lock_release':released,
|
||||
'fresh_preserved':True,'tracked_and_linked_preserved':True,'active_lock_preserved':True})
|
||||
|
||||
|
||||
def s3_outage():
|
||||
c=Client(timeout=8)
|
||||
alice,bob,_,chat=c.bootstrap()
|
||||
raw=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
||||
att=upload(c,alice,chat,raw)['json']
|
||||
try:
|
||||
cluster.compose('stop','-t','2','minio')
|
||||
req=urllib.request.Request(c.base+'/api/v1/chats/'+chat['id']+'/attachments?filename=outage.png',data=raw,method='POST',headers={'Authorization':'Bearer '+alice['token'],'Content-Type':'image/png'})
|
||||
started=time.monotonic()
|
||||
try:
|
||||
response=urllib.request.urlopen(req,timeout=8)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
with response:
|
||||
status=response.code
|
||||
elapsed=time.monotonic()-started
|
||||
check(status in (502,503,504) and elapsed<8,'S3 outage was not bounded/safe')
|
||||
finally:
|
||||
cluster.compose('start','minio')
|
||||
cluster.wait_healthy(cluster.container_ids('minio')[0])
|
||||
cluster.wait_cluster()
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original lost after S3 restart')
|
||||
save('s3-outage.json',{'result':'PASS','upload_status_during_outage':status,'response_seconds':round(elapsed,4),
|
||||
'original_preserved_after_restart':True,'same_session_after_recovery':True})
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['private','failover','interrupted','cleanup','outage','all'],default='all',nargs='?')
|
||||
args=parser.parse_args()
|
||||
modes={'private':private_bucket,'failover':upload_failover,'interrupted':interrupted_upload,'cleanup':crash_orphan,'outage':s3_outage}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:', name)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
||||
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, concurrency=4):
|
||||
self.c = Client(timeout=6)
|
||||
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
||||
self.concurrency = concurrency
|
||||
self.done = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts = []
|
||||
self.confirmed = {}
|
||||
self.prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(self, number):
|
||||
client = Client(timeout=6)
|
||||
index = 0
|
||||
while not self.done.is_set():
|
||||
text = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
key = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.alice,self.chat,text,key=key)
|
||||
row = response['json']
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
||||
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
||||
with self.lock:
|
||||
self.confirmed[row['id']] = row
|
||||
index += 1
|
||||
except Failure:
|
||||
# Only idempotent message POST is retried, with its original body and key.
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
||||
time.sleep(0.02)
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
|
||||
def start(self):
|
||||
self.started = time.monotonic()
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
for future in self.futures:
|
||||
future.result(timeout=10)
|
||||
self.pool.shutdown()
|
||||
self.ended = time.monotonic()
|
||||
|
||||
def summary(self, start=None, end=None):
|
||||
start = self.started if start is None else start
|
||||
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
||||
with self.lock:
|
||||
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
||||
success = [a for a in attempts if a['ok']]
|
||||
duration = end-start
|
||||
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
||||
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
||||
'successful_rps':round(len(success)/duration,3),
|
||||
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
||||
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
||||
'instances':sorted({a['instance'] for a in success})}
|
||||
|
||||
def verify_data(self):
|
||||
rows = history(self.c,self.bob,self.chat)
|
||||
actual = {row['id']:row for row in rows}
|
||||
check(len(actual)==len(rows),'duplicated message ids')
|
||||
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
||||
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
||||
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
||||
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
||||
|
||||
|
||||
def concurrent_replay():
|
||||
cluster.wait_cluster()
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
def send(_):
|
||||
local = Client()
|
||||
barrier.wait()
|
||||
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send,range(8)))
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
||||
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
||||
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
||||
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
||||
'responses_201':8,'messages_created':1,'different_body_status':409})
|
||||
|
||||
|
||||
def verify_persistence():
|
||||
before = cluster.snapshot()
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
||||
after = cluster.snapshot()
|
||||
roles = ['api','postgres','redis','minio','cleanup','lb']
|
||||
previous = {item['container']:item for item in before}
|
||||
for item in after:
|
||||
if item['service'] in roles:
|
||||
old = previous[item['container']]
|
||||
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
||||
check(old['volumes']==item['volumes'],'volume identities changed')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
||||
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
||||
|
||||
|
||||
def public_failover():
|
||||
cluster.wait_cluster()
|
||||
try:
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
||||
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
||||
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
||||
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
||||
finally:
|
||||
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
||||
cluster.wait_cluster()
|
||||
|
||||
|
||||
def reject_new_connection(survivor, address):
|
||||
code = '''import json,socket,sys
|
||||
try:
|
||||
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
||||
except OSError as exc:
|
||||
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
||||
else:
|
||||
s.close()
|
||||
print(json.dumps({'accepted':True}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
||||
|
||||
|
||||
def failover_each():
|
||||
cluster.wait_cluster()
|
||||
originals = cluster.replicas()
|
||||
check(len(originals)==2,'two running replicas required')
|
||||
traffic = Traffic().start()
|
||||
phases = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
baseline_end = time.monotonic()
|
||||
baseline = traffic.summary(end=baseline_end)
|
||||
for original in originals:
|
||||
# Alias changes on restart, so resolve the current process of this exact container.
|
||||
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
||||
all_before = cluster.inspect([current['container']])[0]
|
||||
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
||||
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
||||
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
||||
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
||||
start = time.monotonic()
|
||||
try:
|
||||
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
||||
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
||||
stop_returned = time.monotonic()
|
||||
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
||||
read_deadline = start+10
|
||||
while True:
|
||||
try:
|
||||
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
||||
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
||||
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
||||
read_done = time.monotonic()
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
||||
time.sleep(0.1)
|
||||
check(read_done-start<=10,'read recovery exceeded10s')
|
||||
rejected = reject_new_connection(survivor,address)
|
||||
check(not rejected['accepted'],'new connection accepted by stopped API')
|
||||
time.sleep(max(0,8-(time.monotonic()-start)))
|
||||
end = time.monotonic()
|
||||
summary = traffic.summary(start,end)
|
||||
with traffic.lock:
|
||||
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
||||
recent = [a for a in successful if a['started']>stop_returned]
|
||||
recovery = min(a['ended'] for a in recent)-start if recent else None
|
||||
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
||||
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
||||
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
||||
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
||||
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
||||
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
||||
'seconds_from_action_start':round(read_done-start,4),
|
||||
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
||||
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
||||
finally:
|
||||
cluster.command(['docker','start',current['container']])
|
||||
cluster.wait_healthy(current['container'])
|
||||
cluster.wait_cluster()
|
||||
time.sleep(2)
|
||||
finally:
|
||||
traffic.stop()
|
||||
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
||||
|
||||
|
||||
def rolling_update():
|
||||
cluster.wait_cluster()
|
||||
before = cluster.replicas()
|
||||
check(len(before)==2,'rolling demonstration expects two replicas')
|
||||
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
target_release = 'lab4-'+uuid.uuid4().hex[:8]
|
||||
traffic = Traffic().start()
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
build_start = time.monotonic()
|
||||
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
||||
build_end = time.monotonic()
|
||||
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
||||
swap_start = time.monotonic()
|
||||
for old in before:
|
||||
step_start = time.monotonic()
|
||||
result = cluster.replace_one(old['container'])
|
||||
result['seconds'] = round(time.monotonic()-step_start,4)
|
||||
replacements.append(result)
|
||||
swap_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
finally:
|
||||
traffic.stop()
|
||||
after = cluster.replicas()
|
||||
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
||||
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
||||
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
||||
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
||||
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
||||
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
||||
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
||||
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
||||
|
||||
|
||||
def database_fixture(container, data):
|
||||
code = '''import json,sys,psycopg
|
||||
from psycopg import sql
|
||||
from messenger.db import setting
|
||||
p=json.load(sys.stdin)
|
||||
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
||||
if p['action']=='install':
|
||||
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
||||
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif p['action']=='remove':
|
||||
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
||||
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
else:
|
||||
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(response.stdout)
|
||||
|
||||
|
||||
def graceful_shutdown():
|
||||
cluster.wait_cluster()
|
||||
nodes = cluster.replicas()
|
||||
target,survivor = nodes[0],nodes[1]
|
||||
address = cluster.front_address(target['container'])
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
marker = 'drain_'+uuid.uuid4().hex
|
||||
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
||||
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
||||
'''
|
||||
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
||||
def in_flight():
|
||||
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(result.stdout)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(in_flight)
|
||||
try:
|
||||
deadline = time.monotonic()+3
|
||||
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
||||
time.sleep(0.05)
|
||||
signaled = time.monotonic()
|
||||
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
||||
deadline = signaled+1.5
|
||||
while True:
|
||||
rejected = reject_new_connection(survivor['container'],address)
|
||||
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
||||
if not rejected['accepted']:
|
||||
break
|
||||
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
||||
time.sleep(0.05)
|
||||
check(still_running,'connection refusal observed only after process exit, not during drain')
|
||||
drained = future.result(timeout=8)
|
||||
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
||||
check(drained['message']['text']==marker,'in-flight body changed')
|
||||
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
||||
elapsed = time.monotonic()-signaled
|
||||
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
||||
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
||||
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
||||
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
||||
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
||||
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
||||
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
||||
'confirmed_message_preserved':True}
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('graceful-shutdown.json',result)
|
||||
|
||||
|
||||
def infrastructure():
|
||||
state = cluster.snapshot()
|
||||
for item in state:
|
||||
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
||||
if item['service']!='lb':
|
||||
check(not item['published_ports'],'service exposed a port bypassing LB')
|
||||
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
||||
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
||||
save('infrastructure.json',{'result':'PASS','containers':state})
|
||||
|
||||
|
||||
def dependency_failures():
|
||||
c = Client(timeout=6)
|
||||
actor = c.user('dependencies')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result = {'result':'PASS'}
|
||||
try:
|
||||
cluster.compose('stop','-t','2','redis')
|
||||
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
||||
finally:
|
||||
cluster.compose('start','redis')
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
def status(path, token=None):
|
||||
headers = {'Authorization':'Bearer '+token} if token else {}
|
||||
request = urllib.request.Request(c.base+path,headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = urllib.request.urlopen(request,timeout=6)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.code, round(time.monotonic()-started,4)
|
||||
try:
|
||||
cluster.compose('stop','-t','2','postgres')
|
||||
code,elapsed = status('/api/v1/users/me',actor['token'])
|
||||
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
||||
anonymous,_ = status('/api/v1/users/me')
|
||||
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
||||
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
||||
check(all(code==200 for code in live),'API process liveness depends on DB')
|
||||
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
||||
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
||||
finally:
|
||||
cluster.compose('start','postgres')
|
||||
cluster.wait_cluster()
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result['same_session_after_recovery'] = True
|
||||
save('dependency-failures.json',result)
|
||||
|
||||
|
||||
def verify_nginx():
|
||||
state = cluster.snapshot()
|
||||
lb = next(item for item in state if item['service']=='lb')
|
||||
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
||||
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
||||
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
||||
c = Client()
|
||||
response = c.call('GET','/health/ready',schema='Health')
|
||||
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
||||
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
||||
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
||||
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
||||
args = parser.parse_args()
|
||||
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
||||
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,297 @@
|
||||
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from redis.exceptions import RedisError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .db import Database
|
||||
from .errors import ApiError, invalid, not_found, unauthorized
|
||||
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
|
||||
from .messages import store_message, validate_key, MESSAGE_FIELDS
|
||||
from .object_store import ObjectStore, unavailable
|
||||
from . import attachments
|
||||
from urllib.parse import quote
|
||||
from .pagination import decode_cursor, page, page_limit, public, stamp
|
||||
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
USER_FIELDS = 'id, username, display_name, created_at'
|
||||
|
||||
|
||||
def error_response(request, error):
|
||||
return JSONResponse({'error': {'code': error.code, 'message': error.message,
|
||||
'request_id': request.state.request_id}},
|
||||
status_code=error.status, headers=error.headers)
|
||||
|
||||
|
||||
def authenticate(request: Request):
|
||||
token = bearer(request.headers.get('Authorization', ''))
|
||||
token_hash = digest(token)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
|
||||
(token_hash,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
request.state.token_hash = token_hash
|
||||
return row['user_id']
|
||||
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
|
||||
|
||||
def chat_public(conn, row):
|
||||
item = public(row)
|
||||
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
|
||||
return item
|
||||
|
||||
|
||||
def create_app():
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.db = Database()
|
||||
app.state.storage = ObjectStore()
|
||||
app.state.upload_slots = asyncio.Semaphore(6)
|
||||
app.state.db.open()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.storage.close()
|
||||
app.state.db.close()
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.state.release = os.environ.get('APP_RELEASE', 'lab4-v1')
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
path = request.url.path
|
||||
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
|
||||
if request.method == 'POST' and json_endpoint:
|
||||
media = request.headers.get('Content-Type', '').split(';')
|
||||
unsupported = media[0].strip().lower() != 'application/json'
|
||||
for parameter in media[1:]:
|
||||
name, _, value = parameter.strip().partition('=')
|
||||
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
|
||||
unsupported = True
|
||||
if unsupported:
|
||||
if path != '/api/v1/users':
|
||||
await run_in_threadpool(authenticate, request)
|
||||
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
|
||||
try:
|
||||
json.loads((await request.body()).decode('utf-8'))
|
||||
except (UnicodeError, json.JSONDecodeError):
|
||||
raise invalid() from None
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
|
||||
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
|
||||
except Exception as exc:
|
||||
# Log exception class only: credentials, SQL and message content stay out of logs.
|
||||
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
|
||||
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = app.state.instance_id
|
||||
response.headers['X-App-Release'] = app.state.release
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, exc):
|
||||
return error_response(request, exc)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error_response(request, invalid())
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error_response(request, not_found() if exc.status_code == 404 else invalid())
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready(request: Request):
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
|
||||
conn.execute('SELECT 1 FROM attachments LIMIT 0')
|
||||
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
|
||||
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
|
||||
request.app.state.storage.check()
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
def register(body: CreateUser, request: Request):
|
||||
password_hash = HASHER.hash(body.password)
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
|
||||
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
return public(row)
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def create_session(request: Request):
|
||||
# Run hashing and synchronous database calls outside the event loop.
|
||||
username, password = basic_credentials(request.headers.get('Authorization', ''))
|
||||
if await request.body():
|
||||
raise invalid()
|
||||
return await run_in_threadpool(login, request, username, password)
|
||||
|
||||
def login(request, username, password):
|
||||
db = request.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
|
||||
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
|
||||
if not row or not valid:
|
||||
raise unauthorized(basic=True)
|
||||
row.pop('password_hash')
|
||||
token, expires_at = new_session(db.session_ttl)
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
|
||||
(digest(token), row['id'], expires_at))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def me(request: Request, user_id=Depends(authenticate)):
|
||||
db = request.app.state.db
|
||||
key = 'user:' + str(user_id)
|
||||
try:
|
||||
cached = db.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (RedisError, ValueError):
|
||||
pass
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
result = public(row)
|
||||
try:
|
||||
db.cache.set(key, json.dumps(result), ex=60)
|
||||
except RedisError:
|
||||
pass
|
||||
return result
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def logout(request: Request, user_id=Depends(authenticate)):
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
|
||||
if str(user_id) not in body.member_ids:
|
||||
raise invalid()
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
|
||||
(uuid4(), body.title)).fetchone()
|
||||
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
|
||||
[(row['id'], uuid_value(member)) for member in body.member_ids])
|
||||
result = chat_public(conn, row)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats(request: Request, user_id=Depends(authenticate)):
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'chats:' + str(user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
|
||||
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
|
||||
result = page(scope, rows, limit)
|
||||
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
|
||||
return chat_public(conn, row)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
key = validate_key(request.headers.get('Idempotency-Key', ''))
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
|
||||
return public(row)
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'messages:' + str(cid)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
|
||||
params = (cid, *after, limit+1) if after else (cid, limit+1)
|
||||
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
|
||||
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
|
||||
return page(scope, rows, limit)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=201)
|
||||
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
|
||||
cid=uuid_value(chat_id)
|
||||
def access():
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn,cid,user_id)
|
||||
await run_in_threadpool(access)
|
||||
return await attachments.upload(request,user_id,cid,filename)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
return attachments.metadata(row)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
if variant!='original':
|
||||
raise invalid()
|
||||
opened=request.app.state.storage.get(row['object_key'])
|
||||
if opened.length!=row['size_bytes']:
|
||||
opened.close()
|
||||
raise unavailable()
|
||||
return attachments.ObjectResponse(opened,media_type=row['content_type'],headers={
|
||||
'Content-Length':str(row['size_bytes']),
|
||||
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Bounded binary upload, image validation and DB/S3 coordination."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
import warnings
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .pagination import public
|
||||
|
||||
MAX_BYTES=10485760
|
||||
MAX_PIXELS=40000000
|
||||
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
|
||||
PngImagePlugin.MAX_TEXT_CHUNK=1048576
|
||||
PngImagePlugin.MAX_TEXT_MEMORY=1048576
|
||||
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
|
||||
DECODER=threading.BoundedSemaphore(1)
|
||||
LOG=logging.getLogger('messenger')
|
||||
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at'
|
||||
|
||||
|
||||
def object_lock(conn,key,wait=True):
|
||||
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
||||
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
|
||||
return True if wait else row['locked']
|
||||
|
||||
|
||||
def filename_value(value):
|
||||
if not value or not value.strip() or len(value)>255:
|
||||
raise invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise invalid() from None
|
||||
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def metadata(row):
|
||||
result=public({key:value for key,value in row.items() if key!='object_key'})
|
||||
result.update(status='ready',variants=[],error_code=None)
|
||||
return result
|
||||
|
||||
|
||||
def validate_image(file,mime):
|
||||
if not DECODER.acquire(timeout=3):
|
||||
raise ApiError(503,'unavailable','Обработка изображений занята')
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
|
||||
if actual is None or actual!=mime:
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
|
||||
if image.width*image.height>MAX_PIXELS:
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
image.load()
|
||||
file.seek(0)
|
||||
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
|
||||
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
|
||||
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
|
||||
finally:
|
||||
DECODER.release()
|
||||
|
||||
|
||||
def compensate(db,store,key):
|
||||
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,key,wait=False):
|
||||
return
|
||||
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(key,)).fetchone()
|
||||
if not exists:
|
||||
store.delete(key)
|
||||
except Exception as exc:
|
||||
# A periodic, grace-protected janitor will retry after dependency recovery.
|
||||
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
|
||||
|
||||
|
||||
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
|
||||
key='originals/'+uuid4().hex
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
object_lock(conn,key)
|
||||
store.put(key,file,length,mime)
|
||||
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key) '
|
||||
'VALUES (%s,%s,%s,%s,%s,%s,%s,%s) RETURNING '+FIELDS,
|
||||
(uuid4(),chat_id,user_id,filename,mime,length,digest,key)).fetchone()
|
||||
return metadata(row)
|
||||
except ApiError:
|
||||
compensate(db,store,key)
|
||||
raise
|
||||
except psycopg.Error:
|
||||
compensate(db,store,key)
|
||||
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
|
||||
except BaseException:
|
||||
compensate(db,store,key)
|
||||
raise
|
||||
|
||||
|
||||
async def upload(request,user_id,chat_id,filename):
|
||||
filename=filename_value(filename)
|
||||
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
|
||||
if mime not in ('image/png','image/jpeg'):
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
|
||||
declared=request.headers.get('Content-Length')
|
||||
if declared is not None:
|
||||
try:
|
||||
length=int(declared)
|
||||
if length<0:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid() from None
|
||||
if length>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
try:
|
||||
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
|
||||
except TimeoutError:
|
||||
raise ApiError(503,'unavailable','Слишком много загрузок') from None
|
||||
try:
|
||||
with tempfile.TemporaryFile(mode='w+b') as file:
|
||||
total=0
|
||||
digest=hashlib.sha256()
|
||||
async def read_body():
|
||||
nonlocal total
|
||||
async for chunk in request.stream():
|
||||
total+=len(chunk)
|
||||
if total>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(file.write,chunk)
|
||||
try:
|
||||
await asyncio.wait_for(read_body(),timeout=30)
|
||||
except TimeoutError:
|
||||
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
|
||||
except ClientDisconnect:
|
||||
raise invalid() from None
|
||||
await run_in_threadpool(validate_image,file,mime)
|
||||
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
|
||||
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
def lookup(conn,attachment_id,user_id):
|
||||
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
|
||||
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
|
||||
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
class ObjectResponse(StreamingResponse):
|
||||
def __init__(self,opened,**kwargs):
|
||||
self.opened=opened
|
||||
super().__init__(opened.chunks(),**kwargs)
|
||||
|
||||
async def __call__(self,scope,receive,send):
|
||||
try:
|
||||
await super().__call__(scope,receive,send)
|
||||
finally:
|
||||
self.opened.close()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
|
||||
import argparse
|
||||
from datetime import datetime,timezone,timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from .attachments import object_lock
|
||||
from .db import Database
|
||||
from .object_store import ObjectStore
|
||||
|
||||
|
||||
def sweep(db,store,grace_seconds=300,apply=False):
|
||||
if grace_seconds<60:
|
||||
raise ValueError('Grace must be at least60seconds')
|
||||
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
|
||||
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
|
||||
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
|
||||
for page in store.pages():
|
||||
for obj in page.get('Contents',[]):
|
||||
result['listed']+=1
|
||||
if obj['LastModified']>=cutoff:
|
||||
result['fresh']+=1
|
||||
continue
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,obj['Key'],wait=False):
|
||||
result['locked']+=1
|
||||
continue
|
||||
if conn.execute('SELECT 1 FROM attachments WHERE object_key=%s',(obj['Key'],)).fetchone():
|
||||
result['tracked']+=1
|
||||
continue
|
||||
result['orphans']+=1
|
||||
if apply:
|
||||
store.delete(obj['Key'])
|
||||
result['deleted']+=1
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--grace-seconds',type=int,default=300)
|
||||
parser.add_argument('--apply',action='store_true')
|
||||
parser.add_argument('--dry-run',action='store_true')
|
||||
parser.add_argument('--loop',action='store_true')
|
||||
parser.add_argument('--interval',type=int,default=60)
|
||||
args=parser.parse_args()
|
||||
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
|
||||
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
|
||||
db=Database()
|
||||
store=ObjectStore()
|
||||
db.open()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
result=sweep(db,store,args.grace_seconds,args.apply)
|
||||
print(json.dumps(result),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
|
||||
if not args.loop:
|
||||
raise
|
||||
finally:
|
||||
Path('/tmp/cleanup-heartbeat').touch()
|
||||
if not args.loop:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Bounded dependencies; PostgreSQL is always the source of truth."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
def setting(name, default=None):
|
||||
filename = os.environ.get(name + '_FILE')
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
|
||||
if value is None:
|
||||
raise RuntimeError(f'Missing configuration: {name}')
|
||||
return value
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.session_ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
self.pool = ConnectionPool(
|
||||
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
|
||||
timeout=2, max_waiting=16, open=False,
|
||||
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
|
||||
)
|
||||
self.cache = Redis.from_url(
|
||||
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=30)
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
@@ -0,0 +1,19 @@
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid():
|
||||
return ApiError(400, 'invalid_request', 'Неверный запрос')
|
||||
|
||||
|
||||
def unauthorized(basic=False):
|
||||
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
|
||||
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
|
||||
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, 'not_found', 'Объект не найден')
|
||||
@@ -0,0 +1,52 @@
|
||||
"""One PostgreSQL transaction owns both a message and its retry result."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from uuid import uuid4
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def validate_key(key):
|
||||
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
raise invalid()
|
||||
return key
|
||||
|
||||
|
||||
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
|
||||
|
||||
|
||||
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
|
||||
attachment_ids=[] if attachment_ids is None else attachment_ids
|
||||
payload={'text':text}
|
||||
if attachment_ids:
|
||||
payload['attachment_ids']=attachment_ids
|
||||
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
params = (user_id, chat_id, key)
|
||||
# A competing INSERT waits for the other transaction, then the next statement sees its result.
|
||||
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
|
||||
(*params, fingerprint))
|
||||
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
|
||||
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
|
||||
if record['message_id'] is not None and record['active']:
|
||||
if record['request_hash'] != fingerprint:
|
||||
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
|
||||
(record['message_id'],)).fetchone()
|
||||
ids=[uuid_value(item) for item in attachment_ids]
|
||||
if ids:
|
||||
allowed=conn.execute('SELECT id FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
|
||||
if len(allowed)!=len(ids):
|
||||
raise not_found()
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
|
||||
if ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
|
||||
[(row['id'],aid,i) for i,aid in enumerate(ids)])
|
||||
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
|
||||
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from .db import setting
|
||||
|
||||
|
||||
def main():
|
||||
directory = Path(__file__).resolve().parents[2] / 'migrations'
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(directory.glob('*.sql')):
|
||||
sql = path.read_text()
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
|
||||
if found:
|
||||
if found[0] != checksum:
|
||||
raise RuntimeError('Applied migration checksum changed: ' + path.name)
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
|
||||
print('Applied', path.name)
|
||||
print('Migrations up to date')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from .errors import invalid
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
try:
|
||||
# Require the UUID string representation, not integer/coercible values.
|
||||
if not isinstance(value, str) or len(value) != 36:
|
||||
raise ValueError
|
||||
return UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', strict=True)
|
||||
|
||||
@field_validator('*')
|
||||
@classmethod
|
||||
def valid_strings(cls, value, info):
|
||||
if isinstance(value, str):
|
||||
if not value.strip() and info.field_name != 'text':
|
||||
raise ValueError('Blank string')
|
||||
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
|
||||
if '\x00' in value:
|
||||
raise ValueError('NUL is not supported')
|
||||
value.encode('utf-8')
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('username')
|
||||
@classmethod
|
||||
def username_ascii(cls, value):
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
raise ValueError('Invalid username')
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def members(cls, value):
|
||||
ids = [uuid_value(item) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('Duplicate member')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def attachment_values(cls,value):
|
||||
ids=[uuid_value(item) for item in value]
|
||||
if len(ids)!=len(set(ids)):
|
||||
raise ValueError('Duplicate attachment')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
@model_validator(mode='after')
|
||||
def nonempty_message(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('Message must contain text or attachment')
|
||||
return self
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
|
||||
from contextlib import contextmanager
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from .db import setting
|
||||
from .errors import ApiError
|
||||
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
def unavailable():
|
||||
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
|
||||
|
||||
|
||||
class OpenObject:
|
||||
def __init__(self,body,release,length):
|
||||
self.body=body
|
||||
self.release=release
|
||||
self.length=length
|
||||
self.closed=False
|
||||
self.close_lock=threading.Lock()
|
||||
|
||||
def chunks(self):
|
||||
try:
|
||||
while True:
|
||||
data=self.body.read(65536)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
with self.close_lock:
|
||||
if self.closed:
|
||||
return
|
||||
self.closed=True
|
||||
try:
|
||||
self.body.close()
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.bucket=setting('S3_BUCKET','messenger-attachments')
|
||||
self.gate=threading.BoundedSemaphore(8)
|
||||
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
|
||||
region_name=setting('S3_REGION','us-east-1'),
|
||||
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
|
||||
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
|
||||
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
|
||||
|
||||
@contextmanager
|
||||
def operation(self):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
yield
|
||||
except (BotoCoreError,ClientError):
|
||||
raise unavailable() from None
|
||||
finally:
|
||||
self.gate.release()
|
||||
|
||||
def check(self):
|
||||
with self.operation():
|
||||
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
|
||||
|
||||
def put(self,key,body,length,mime):
|
||||
with self.operation():
|
||||
body.seek(0)
|
||||
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
|
||||
|
||||
def delete(self,key):
|
||||
with self.operation():
|
||||
self.client.delete_object(Bucket=self.bucket,Key=key)
|
||||
|
||||
def get(self,key):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
result=self.client.get_object(Bucket=self.bucket,Key=key)
|
||||
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
|
||||
except (BotoCoreError,ClientError):
|
||||
self.gate.release()
|
||||
raise unavailable() from None
|
||||
except BaseException:
|
||||
self.gate.release()
|
||||
raise
|
||||
|
||||
def pages(self):
|
||||
token=None
|
||||
while True:
|
||||
params={'Bucket':self.bucket,'Prefix':PREFIX,'MaxKeys':1000}
|
||||
if token:
|
||||
params['ContinuationToken']=token
|
||||
with self.operation():
|
||||
result=self.client.list_objects_v2(**params)
|
||||
yield result
|
||||
if not result.get('IsTruncated'):
|
||||
break
|
||||
token=result['NextContinuationToken']
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,56 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
from .errors import invalid
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def encode_cursor(scope, row):
|
||||
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(scope, value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
|
||||
raise ValueError
|
||||
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
|
||||
raise ValueError
|
||||
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
|
||||
if ts.tzinfo is None:
|
||||
raise ValueError
|
||||
return ts, uuid_value(data[2])
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
def page_limit(value):
|
||||
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
|
||||
raise invalid()
|
||||
return int(value)
|
||||
|
||||
|
||||
def public(row):
|
||||
def value(item):
|
||||
if isinstance(item,datetime):
|
||||
return stamp(item)
|
||||
if isinstance(item,UUID):
|
||||
return str(item)
|
||||
if isinstance(item,list):
|
||||
return [value(part) for part in item]
|
||||
return item
|
||||
return {key:value(item) for key,item in row.items()}
|
||||
|
||||
|
||||
def page(scope, rows, limit):
|
||||
return {'items': [public(row) for row in rows[:limit]],
|
||||
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
|
||||
@@ -0,0 +1,49 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, InvalidHashError
|
||||
from .errors import unauthorized
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
|
||||
|
||||
|
||||
def digest(token):
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, encoded = header.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
return username, password
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized(basic=True) from None
|
||||
|
||||
|
||||
def check_password(encoded, password):
|
||||
try:
|
||||
return HASHER.verify(encoded, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def bearer(header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError
|
||||
return token
|
||||
except ValueError:
|
||||
raise unauthorized() from None
|
||||
|
||||
|
||||
def new_session(ttl):
|
||||
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from messenger.app import create_app
|
||||
|
||||
PASSWORD = 'Тестовый:пароль🔐' * 8
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with TestClient(create_app()) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def user(c):
|
||||
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
|
||||
r = c.post('/api/v1/users', json=body)
|
||||
assert r.status_code == 201, r.text
|
||||
return r.json(), body
|
||||
|
||||
|
||||
def login(c, body):
|
||||
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
|
||||
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
|
||||
assert r.status_code == 201, r.text
|
||||
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
|
||||
|
||||
|
||||
def test_auth_unicode_logout_and_cache(client):
|
||||
u, body = user(client)
|
||||
h, session = login(client, body)
|
||||
assert session['user'] == u
|
||||
assert len(session['token']) >= 43
|
||||
key = 'user:' + u['id']
|
||||
client.app.state.db.cache.delete(key)
|
||||
for _ in range(2):
|
||||
assert client.get('/api/v1/users/me', headers=h).json() == u
|
||||
cached = json.loads(client.app.state.db.cache.get(key))
|
||||
assert cached == u
|
||||
assert 0 < client.app.state.db.cache.ttl(key) <= 60
|
||||
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
|
||||
r = client.get('/api/v1/users/me', headers=h)
|
||||
assert r.status_code == 401
|
||||
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
|
||||
assert r.json()['error']['request_id'] == r.headers['x-request-id']
|
||||
|
||||
|
||||
def test_session_expiry(monkeypatch):
|
||||
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
|
||||
with TestClient(create_app()) as client:
|
||||
_, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 200
|
||||
time.sleep(2.1)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 401
|
||||
|
||||
|
||||
def test_registration_race(client):
|
||||
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
|
||||
with ThreadPoolExecutor(max_workers=6) as executor:
|
||||
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
|
||||
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
|
||||
|
||||
|
||||
def test_chat_rollback_after_insert(client):
|
||||
u, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
title = 'rollback_' + uuid.uuid4().hex
|
||||
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
|
||||
assert r.status_code == 400
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
|
||||
|
||||
|
||||
def test_cursor_access_and_strict_validation(client):
|
||||
a, ab = user(client)
|
||||
b, bb = user(client)
|
||||
ah, _ = login(client, ab)
|
||||
bh, _ = login(client, bb)
|
||||
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
|
||||
path = '/api/v1/chats/' + chats[0]['id']
|
||||
for text in ['Один 👋', 'Два', 'Три']:
|
||||
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
|
||||
p = client.get(path+'/messages?limit=2', headers=ah).json()
|
||||
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
|
||||
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
|
||||
assert [m['text'] for m in tail['items']] == ['Три']
|
||||
assert tail['next_cursor'] is None
|
||||
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
|
||||
assert r.status_code == 400
|
||||
assert client.get(path, headers=bh).status_code == 404
|
||||
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
|
||||
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
|
||||
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
|
||||
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
|
||||
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
|
||||
assert client.post('/api/v1/users',json=payload).status_code == 400
|
||||
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
|
||||
|
||||
|
||||
def test_session_storage_and_migrations(client):
|
||||
from messenger.migrate import main as migrate
|
||||
u, body = user(client)
|
||||
_, session = login(client, body)
|
||||
db = client.app.state.db
|
||||
def snapshot():
|
||||
with db.pool.connection() as conn:
|
||||
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
|
||||
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
|
||||
before = snapshot()
|
||||
migrate()
|
||||
migrate()
|
||||
assert snapshot() == before
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
|
||||
assert row['token_hash'] != session['token']
|
||||
assert len(row['token_hash']) == 64
|
||||
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
|
||||
assert stored.startswith('$argon2id$')
|
||||
assert stored != body['password']
|
||||
|
||||
|
||||
def test_json_media_type_and_utf8(client):
|
||||
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
|
||||
raw = json.dumps(body).encode()
|
||||
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
|
||||
r = client.post('/api/v1/users',content=raw,headers=headers)
|
||||
assert r.status_code == 415
|
||||
assert r.json()['error']['code'] == 'unsupported_media_type'
|
||||
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
|
||||
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
|
||||
assert r.status_code == 400, encoding
|
||||
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
|
||||
assert r.status_code == 401
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Real PostgreSQL + private S3 integration; synthetic images only."""
|
||||
import hashlib
|
||||
import io
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from PIL import Image
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def image_bytes(kind='PNG'):
|
||||
stream = io.BytesIO()
|
||||
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def actors(c):
|
||||
a,ab=user(c)
|
||||
b,bb=user(c)
|
||||
e,eb=user(c)
|
||||
ah,_=login(c,ab)
|
||||
bh,_=login(c,bb)
|
||||
eh,_=login(c,eb)
|
||||
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
|
||||
return ah,bh,eh,chat
|
||||
|
||||
|
||||
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
|
||||
raw=image_bytes() if raw is None else raw
|
||||
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
|
||||
headers={**headers,'Content-Type':mime})
|
||||
|
||||
|
||||
def test_original_png_jpeg_and_message(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
|
||||
raw=image_bytes(kind)
|
||||
r=upload(client,ah,chat,raw,mime)
|
||||
assert r.status_code==201
|
||||
att=r.json()
|
||||
assert att['status']=='ready' and att['variants']==[] and att['error_code'] is None
|
||||
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
|
||||
assert 'object_key' not in att and 'bucket' not in att
|
||||
path='/api/v1/attachments/'+att['id']
|
||||
assert client.get(path,headers=bh).json()==att
|
||||
content=client.get(path+'/content',headers=bh)
|
||||
assert content.status_code==200 and content.content==raw
|
||||
assert content.headers['Content-Type']==mime
|
||||
for suffix in ['', '/content']:
|
||||
assert client.get(path+suffix,headers=eh).status_code==404
|
||||
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==400
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
|
||||
json={'text':'','attachment_ids':[att['id']]})
|
||||
assert response.status_code==201
|
||||
assert response.json()['attachment_ids']==[att['id']]
|
||||
|
||||
|
||||
def test_invalid_media_corruption_limits_and_filename(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
|
||||
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
|
||||
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
|
||||
for raw,mime,expected in cases:
|
||||
response=upload(client,ah,chat,raw,mime)
|
||||
assert response.status_code==expected
|
||||
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
|
||||
# Valid PNG header advertises >40Mp; reject before attempting decompression.
|
||||
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
|
||||
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
|
||||
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
|
||||
assert upload(client,ah,chat,bomb).status_code==413
|
||||
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
|
||||
assert upload(client,ah,chat,filename=filename).status_code==400
|
||||
|
||||
|
||||
def test_foreign_attachment_binding_and_normalized_replay(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
|
||||
path='/api/v1/chats/'+other['id']+'/messages'
|
||||
for aid in [att['id'],str(uuid.uuid4())]:
|
||||
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
|
||||
assert response.status_code==404
|
||||
assert upload(client,eh,chat).status_code==404
|
||||
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
first=client.post(path,headers=h,json={'text':'same'})
|
||||
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
|
||||
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
|
||||
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
|
||||
assert conflict.status_code==409
|
||||
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
|
||||
assert duplicates.status_code==400
|
||||
|
||||
|
||||
def test_compensate_s3_when_metadata_insert_fails(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
db=client.app.state.db
|
||||
store=client.app.state.storage
|
||||
before={item['Key'] for page in store.pages() for item in page.get('Contents',[])}
|
||||
function='fail_attachment_'+uuid.uuid4().hex
|
||||
trigger='fail_attachment_'+uuid.uuid4().hex
|
||||
filename='fault_'+uuid.uuid4().hex+'.png'
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
response=upload(client,ah,chat,filename=filename)
|
||||
assert response.status_code==503
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
|
||||
after={item['Key'] for page in store.pages() for item in page.get('Contents',[])}
|
||||
assert after==before
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
|
||||
|
||||
def test_compensation_preserves_committed_object(client):
|
||||
from messenger.attachments import compensate
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
# A retry after an unknown COMMIT result must never delete an already tracked object.
|
||||
compensate(client.app.state.db,client.app.state.storage,key)
|
||||
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
|
||||
assert content.status_code==200 and content.content==image_bytes()
|
||||
|
||||
|
||||
def test_stream_close_even_if_headers_cannot_be_sent(client):
|
||||
import asyncio
|
||||
import pytest
|
||||
from messenger.attachments import ObjectResponse
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
store=client.app.state.storage
|
||||
opened=store.get(key)
|
||||
response=ObjectResponse(opened,media_type='image/png')
|
||||
async def fail_send(message):
|
||||
raise OSError('synthetic disconnected transport')
|
||||
async def receive():
|
||||
return {'type':'http.disconnect'}
|
||||
with pytest.raises(Exception):
|
||||
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
|
||||
assert opened.closed
|
||||
acquired=0
|
||||
try:
|
||||
for _ in range(8):
|
||||
assert store.gate.acquire(blocking=False)
|
||||
acquired+=1
|
||||
finally:
|
||||
for _ in range(acquired):
|
||||
store.gate.release()
|
||||
|
||||
|
||||
def test_chunked_upload_is_limited_without_content_length(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
chunks=iter([b'x'*1048576]*10+[b'x'])
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
|
||||
content=chunks,headers={**ah,'Content-Type':'image/png'})
|
||||
assert response.status_code==413
|
||||
assert response.json()['error']['code']=='payload_too_large'
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def setup_chat(c, extra=None):
|
||||
u, body = user(c)
|
||||
headers, _ = login(c, body)
|
||||
members = [u['id']] + (extra or [])
|
||||
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
|
||||
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
|
||||
|
||||
|
||||
def test_instance_alias_on_success_and_errors(client):
|
||||
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
|
||||
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
|
||||
ids = [r.headers.get('X-Instance-Id') for r in responses]
|
||||
assert ids[0] and len(set(ids)) == 1
|
||||
assert all(r.headers.get('X-Request-Id') for r in responses)
|
||||
|
||||
|
||||
def test_key_required_replay_and_conflict(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
|
||||
for key in ['', 'a b', 'bad/', 'x'*129]:
|
||||
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
|
||||
h = {**headers, 'Idempotency-Key':'valid:._-123'}
|
||||
first = client.post(path, headers=h, json={'text':'same'})
|
||||
assert first.status_code == 201
|
||||
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
|
||||
assert replay.status_code == 201 and replay.json() == first.json()
|
||||
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
|
||||
assert len(client.get(path,headers=headers).json()['items']) == 1
|
||||
|
||||
|
||||
def test_concurrent_key_produces_one_message(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
|
||||
assert [r.status_code for r in results] == [201]*16
|
||||
assert all(r.json() == results[0].json() for r in results)
|
||||
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
|
||||
|
||||
|
||||
def test_scope_and_replay_rechecks_authorization(client):
|
||||
b, bb = user(client)
|
||||
bh, _ = login(client, bb)
|
||||
a, ah, path = setup_chat(client, [b['id']])
|
||||
key = 'same-in-independent-scopes'
|
||||
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
|
||||
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
|
||||
other_path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
|
||||
assert third.status_code == 201 and third.json()['id'] != first['id']
|
||||
outsider, ob = user(client)
|
||||
oh, _ = login(client, ob)
|
||||
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
|
||||
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
|
||||
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
|
||||
assert 86390 <= ttl <= 86400
|
||||
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
|
||||
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
assert len(client.get(path,headers=h).json()['items']) == 2
|
||||
|
||||
|
||||
def test_rollback_side_effect_and_key_on_database_error(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'fault_' + uuid.uuid4().hex
|
||||
function = 'fault_' + uuid.uuid4().hex
|
||||
trigger = 'trigger_' + uuid.uuid4().hex
|
||||
db = client.app.state.db
|
||||
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert failed.status_code == 500
|
||||
assert failed.headers['X-Instance-Id']
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert retried.status_code == 201
|
||||
assert client.get(path,headers=h).json()['items'] == [retried.json()]
|
||||
|
||||
|
||||
def test_ready_requires_compatible_schema(client):
|
||||
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
|
||||
try:
|
||||
r = client.get('/health/ready')
|
||||
assert r.status_code == 503
|
||||
assert r.json()['error']['code'] == 'unavailable'
|
||||
assert client.get('/health/live').status_code == 200
|
||||
finally:
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
|
||||
Reference in New Issue
Block a user