ЛР3 - БВТ2302 - Осокин - на 4 #91
@@ -17,4 +17,5 @@ backups
|
||||
__pycache__
|
||||
node_modules
|
||||
evidence
|
||||
output
|
||||
.DS_Store
|
||||
|
||||
+3
-1
@@ -7,7 +7,9 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Shared across API replicas, generated by scripts/bootstrap.py:
|
||||
CURSOR_KEY_FILE=/run/secrets/cursor_key
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
@@ -28,5 +28,9 @@ coverage/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
output/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
# Local study notes; keep outside published Git history.
|
||||
/DEFENSE.md
|
||||
/MR_DRAFT.md
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
FROM python:3.13.7-slim-bookworm
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
COPY contracts ./contracts
|
||||
COPY tests ./tests
|
||||
COPY student-tests ./student-tests
|
||||
COPY scripts ./scripts
|
||||
COPY lab.json requirements-dev.txt ./
|
||||
USER app
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"]
|
||||
CMD ["python", "-m", "messenger.server"]
|
||||
@@ -4,7 +4,7 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-persistence test-failover verify test-own rolling-update
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@@ -28,3 +28,13 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
verify:
|
||||
$(PYTHON) scripts/verify.py
|
||||
|
||||
test-own:
|
||||
docker compose exec -T api python student-tests/test_idempotency.py
|
||||
$(PYTHON) student-tests/test_recovery.py
|
||||
|
||||
rolling-update:
|
||||
$(PYTHON) scripts/labctl.py rolling
|
||||
|
||||
@@ -1,37 +1,195 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор: **Осокин Глеб**, группа **БВТ2302**, Gitea **glebossokin**.
|
||||
- Целевая оценка: **4**.
|
||||
- Основа: skeleton `0ab3a341aa7ba32363ce43061d4ff02b1f2964af`; ветка `solution/BVT2302-Osokin`.
|
||||
- Перенесены собственные исходники lab2; помощь ИИ использовалась для разбора требований, проверки результатов и подготовки отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Git, Python 3.9+ на клиенте, Docker с Linux containers и Compose v2 с `scale`, `--wait`, `--no-recreate`. Make необязателен. Проверенная среда: клиентский Python 3.14.3, Docker Engine 29.8.2, контейнерный Python **3.13.7**. Образы: `python:3.13.7-slim-bookworm`, `postgres:17.6-bookworm`, `redis:7.4.5-bookworm`, `haproxy:3.0.16-alpine`. [requirements.txt](requirements.txt): psycopg/psycopg-binary 3.2.10, psycopg-pool 3.2.6, redis-py 6.4.0, typing_extensions 4.15.0. Digests зависимостей и параметры машины — [environment.json](evidence/environment.json).
|
||||
|
||||
Из корня репозитория в PowerShell:
|
||||
|
||||
```powershell
|
||||
python -X utf8 scripts/bootstrap.py
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
Invoke-RestMethod http://127.0.0.1:8080/health/ready
|
||||
python -X utf8 scripts/check_contract.py
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python -X utf8 scripts/demo.py
|
||||
python -X utf8 scripts/verify.py
|
||||
```
|
||||
|
||||
`bootstrap.py` генерирует локальный случайный пароль PostgreSQL, DSN, URL Redis и общий ключ HMAC для cursor в `secrets/`, создаёт `.env`. Существующие значения сохраняются. Эти файлы исключены из Git и build context. Сохраняйте secrets между запусками с существующими volumes: замена файла пароля не изменяет пароль уже инициализированной БД. Для другого публичного порта измените `APP_PORT` в `.env` и задайте `$env:BASE_URL='http://127.0.0.1:НОВЫЙ_ПОРТ'`. Измерения сделаны на 8080.
|
||||
|
||||
Compose ждёт health PostgreSQL, выполняет одноразовый `migrate`, после его успешного завершения запускает две API, ждёт их readiness и запускает HAProxy. Миграции не выполняются внутри API. `migrate` использует транзакцию и общий advisory lock, сверяет SHA-256 файлов с `schema_migrations`. Повторный запуск не дублирует данные:
|
||||
|
||||
```powershell
|
||||
docker compose run --rm --no-deps migrate
|
||||
```
|
||||
|
||||
На Linux/macOS используйте `python3` вместо `python -X utf8`. Hooks `.sh` имеют executable bit в Git:
|
||||
|
||||
```sh
|
||||
make check
|
||||
make test BASE_URL=http://127.0.0.1:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 scripts/labctl.py restore
|
||||
make verify
|
||||
```
|
||||
|
||||
Windows-эквиваленты публичных сценариев:
|
||||
|
||||
```powershell
|
||||
python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080
|
||||
python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080
|
||||
python -X utf8 scripts/labctl.py restore
|
||||
```
|
||||
|
||||
Failover намеренно оставляет выбранный API остановленным; `restore` возвращает две реплики. `.cmd` и `.sh` вызывают один `labctl.py`; `TARGET_INSTANCE_ID` приходит от публичного теста и выбирает соответствующий контейнер текущего проекта. Restart перезапускает обе API, PostgreSQL, Redis и LB; ID, PID, StartedAt и имена volumes сохраняются в `evidence/restart.json`.
|
||||
|
||||
**Безопасная остановка:** `docker compose down`. Данные остаются в `mtusi-lab3-osokin_pgdata` и `mtusi-lab3-osokin_redisdata`, повторный `up -d --wait` использует их. **Намеренное удаление всех учебных данных:** `docker compose down --volumes` — удаляет пользователей, чаты, сообщения, результаты идемпотентности и сессии. Проверки эту команду не используют.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → localhost:8080 → HAProxy (round robin)
|
||||
├─ API A ─┬─ PostgreSQL: данные + идемпотентность
|
||||
└─ API B ─┴─ Redis AOF: общие сессии с TTL
|
||||
migrate → PostgreSQL (однократно)
|
||||
```
|
||||
|
||||
API — один Compose-service с `scale: 2`, без `container_name`, sticky session и host-портов. В приватной сети `backend` находятся API, DB, Redis, migrate и backend LB. Только LB подключён к `frontend` и публикует 127.0.0.1:8080. API не имеют data volumes. Пароль DB доступен только DB; DSN и общий cursor key — API/migrate; LB не получает secrets. Выдача migrate также cache/cursor secrets — упрощение общего app-шаблона, для миграций они не нужны. Compose secrets — локальные файлы, не зашифрованный vault.
|
||||
|
||||
**PostgreSQL** — источник истины пользователей, salted scrypt hashes, чатов, участников, сообщений и результатов повторов. Сохранены PK, FK, UNIQUE username, атомарное создание чата и параметризованный SQL. На API: pool min 1 / max 8, ожидание 1 s, максимум 16 ожидающих; connect 2 s, statement 2 s, lock 1 s. Две API имеют суммарно до 16 обычных соединений плюс migrate. История читается с primary без кэша и видит завершённый COMMIT любой API. Индекс `(chat_id, created_at, id)` и HMAC cursor обеспечивают keyset-пагинацию. Поздний COMMIT с ранним timestamp может оказаться до уже пройденного cursor; межзапросный snapshot не обещается.
|
||||
|
||||
**Redis** — единое хранилище сессий. Ключ `session:sha256(token)`, значение UUID пользователя, абсолютный expiry через PXAT. Токен — 256 случайных бит, raw значение не хранится. Чтение и restart не продлевают TTL; logout сразу действует на обеих API. Redis pool max 8 на API, connect/read по 0.5 s, без автоматических retry. AOF `appendfsync always`, именованный volume, лимит данных 64 MiB и `noeviction`. Потеря Redis делает API неготовым, а не анонимно доверенным. Отсутствие Redis-пароля — упрощение изолированного учебного стенда.
|
||||
|
||||
**Идемпотентность:** миграция [002](migrations/002_idempotency.sql) добавляет таблицу с PK `(actor_id, chat_id, key)`. Метод POST и путь сообщения заданы назначением таблицы. Обязательный ключ — ASCII 1–128. Тело нормализуется с пустым `attachment_ids` по умолчанию и хешируется SHA-256; порядок JSON-полей не влияет.
|
||||
|
||||
Одна транзакция `create_message` берёт `pg_advisory_xact_lock` по хешу области запроса, проверяет прежний результат, создаёт сообщение и сохраняет ответ JSONB с expiry. HTTP 201 отправляется после COMMIT. Конкурентная транзакция на другой API ждёт и возвращает тот же Message и 201; другое тело даёт 409. Ошибка до COMMIT откатывает обе записи. Потерянный HTTP-ответ безопасно повторяется. Коллизия 64-битного lock-хеша лишь сериализует независимые операции, идентичность задаёт полный PK. Lock timeout даёт 503 и не фиксируется как результат.
|
||||
|
||||
Сессия и членство проверяются заново перед каждым повтором. Результат удерживается минимум 86400 s; меньший TTL отвергается конфигурацией. Истёкшая запись заменяется при повторном использовании области: после TTL новый POST может создать новое сообщение. Периодической очистки старых неиспользуемых ключей нет; это ограничение роста БД, не отключение истечения гарантии повторов.
|
||||
|
||||
**HAProxy:** round robin, `server-template` до 10 backend slots, Docker DNS `127.0.0.11`, hold valid 1 s. Фактически API два; сравнения 1/2/4 не проводились. Проверка HTTP `/health/ready` каждые 1 s, исключение после двух неудач, возврат после одной успешной проверки. Connect timeout 0.5 s, check timeout 1 s; консервативный бюджет обнаружения зависшего backend до 5 s с интервалами/connect/check при нормальной работе планировщика. Это настройка, не SLA при зависании всего host. Измеренные последние ошибки после stop: 1.963 и 1.620 s.
|
||||
|
||||
`/health/live` проверяет процесс; `/health/ready` — PostgreSQL и Redis, обе зависимости критичны. Docker healthcheck API: interval 2 s, timeout 5 s, retries 2; он нужен Compose для старта. Оперативное исключение трафика делает собственный healthcheck HAProxy. LB timeouts: headers 3 s, queue 1 s, connect 0.5 s, client/server 8 s, `retries 0`. LB не повторяет произвольные POST. Клиент сохраняет key/body неопределённого сообщения. Собственные HTML 503 балансировщика могут не иметь API-заголовков, что допускает контракт; тест учитывает их как ошибки.
|
||||
|
||||
**SIGTERM:** draining прекращает приём, ещё не начатые запросы получают 503, выполняющиеся получают до 6 s на COMMIT и ответ, затем закрываются сокеты/пулы. Compose grace period 10 s; action scripts явно задают `docker stop --time 8`, поэтому в сценариях остановки/замены действует 8 s; API socket timeout 2 s, workers 16, queue 32. Стабильный `X-Instance-Id` включает hostname и случайный суффикс процесса; при рестарте меняется. Каждый ответ API, включая health/error/queue rejection, содержит его и новый `X-Request-Id`. Логи не содержат тел и Authorization.
|
||||
|
||||
| Контейнер | CPU | RAM |
|
||||
| --- | ---: | ---: |
|
||||
| Каждый API / одноразовый migrate | 1 | 256 MiB |
|
||||
| PostgreSQL | 2 | 512 MiB |
|
||||
| Redis | 0.5 | 128 MiB |
|
||||
| HAProxy | 0.5 | 128 MiB |
|
||||
|
||||
Реальные квоты и отсутствие опубликованных API-портов видны в [processes.txt](evidence/processes.txt). API запускается от UID 10001.
|
||||
|
||||
| Переменная `.env.example` | Реальное соответствие |
|
||||
| --- | --- |
|
||||
| APP_HOST | Внутри API фиксирован 0.0.0.0 для LB; пример отражает значение |
|
||||
| APP_PORT | Host port LB на 127.0.0.1, внутренний порт всегда 8080 |
|
||||
| APP_ENV | Маркер окружения lab |
|
||||
| SESSION_TTL_SECONDS | TTL новых сессий, default 3600; smoke/persistence требуют ≥300 |
|
||||
| SESSION_STORE | Реализация и Compose фиксируют redis |
|
||||
| DATABASE_URL_FILE, CACHE_URL_FILE | Пути Compose secrets, значения генерирует bootstrap |
|
||||
| CURSOR_KEY_FILE | Общий постоянный HMAC key для всех API, генерирует bootstrap |
|
||||
| API_REPLICAS | default 2; grade 4 сценарии рассчитаны на две API |
|
||||
| IDEMPOTENCY_TTL_SECONDS | default и минимум 86400 |
|
||||
|
||||
### Поочерёдное обновление и миграции
|
||||
|
||||
```powershell
|
||||
docker compose build api
|
||||
docker compose run --rm --no-deps migrate
|
||||
python -X utf8 scripts/labctl.py rolling
|
||||
# Проверка этой же операции под нагрузкой:
|
||||
python -X utf8 student-tests/test_recovery.py --mode rolling
|
||||
```
|
||||
|
||||
Скрипт сохраняет ID двух исходных API, останавливает и удаляет одну, создаёт недостающую из текущего образа через `up --no-deps --no-recreate`, ждёт health и DNS/LB, затем заменяет вторую исходную API. Живой сосед не пересоздаётся. Compose может увеличить номера реплик: в измерении появились api-3/api-4. [rolling-processes.json](evidence/baseline/rolling-processes.json) подтверждает новые ID/PID обоих контейнеров. Проверена реальная замена контейнеров одной версии/образа, изменение алгоритма приложения не имитировалось.
|
||||
|
||||
Миграция 002 аддитивная и не меняет lab2-таблицы; старая версия может работать с новой схемой. Но смешанный приём сообщений lab2/lab3 не гарантирует новый контракт идемпотентности, поэтому первичный переход выполняется с остановкой старой lab2. Поочерёдные замены проверены между API уже уровня lab3. Произвольные несовместимые миграции требуют expand/contract и отдельной проверки оператора. Если новая API неготова, скрипт не начинает замену второй; автоматического rollback образа нет. Обычный `up --build` — команда запуска, не rolling update.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Команды из корня lab3; весь прогон выполнен без изменения публичных tests/contracts.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| Структура skeleton | `python -X utf8 scripts/check_contract.py` | PASS, refs и синтаксис | [check.txt](evidence/check.txt) |
|
||||
| Запуск из чистого clone | `python -X utf8 scripts/clean_check.py` | Новый clone без .env/secrets, отдельные новые volumes и порт 8081, smoke и 16 конкурентных повторов PASS | [clean-clone.txt](evidence/clean-clone.txt) |
|
||||
| Весь стенд, ≥2 API, единый LB, общие DB/сессии | `docker compose up -d --build --wait`; `python -X utf8 scripts/labctl.py state` | 2 healthy API с разными PID, DB/Redis/LB, только LB публикует порт | [compose.yaml](compose.yaml), [processes.txt](evidence/processes.txt) |
|
||||
| Smoke, минимум lab2 с объявленным переходом | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS lab3 до/после отказов; auth, доступ, pagination, replay/logout | [smoke.txt](evidence/smoke.txt), [smoke-after.txt](evidence/smoke-after.txt) |
|
||||
| Одна сессия на двух реальных процессах | `docker compose exec -T api python student-tests/test_idempotency.py` | Один Bearer работает на двух IP/X-Instance-Id | [idempotency.txt](evidence/idempotency.txt) |
|
||||
| Миграции отдельно, повтор безопасен | `docker compose run --rm --no-deps migrate` | Обе миграции Already applied, без дублей | [migration-repeat.txt](evidence/migration-repeat.txt) |
|
||||
| Persistence API и всех хранилищ | public resilience `--mode persistence` + restart hook | Прежние токен/данные сохранены, StartedAt изменились, volumes те же | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/baseline/restart.json) |
|
||||
| Публичный failover ≤10 s, ≥2 наблюдаемых API | public resilience `--mode failover` + stop-one hook | 112 успешных записей, 2 ошибки, recovery 1.852 s, без потерь/дублей | [failover.txt](evidence/failover.txt), [stop-one.json](evidence/baseline/stop-one.json) |
|
||||
| Конкурентная общая идемпотентность | 16 одновременно стартующих POST к двум API | 16 одинаковых ответов, одна запись; другое тело 409, независимые области, logout → 401 | [idempotency.txt](evidence/idempotency.txt) |
|
||||
| Readiness/timeouts/SIGTERM, исключение backend | Два stop-сценария под чтением/записью | LB продолжает через соседа; последние ошибки ≤1.963 s | [haproxy.cfg](infra/haproxy.cfg), [load-grade4.txt](evidence/load-grade4.txt) |
|
||||
| Поочерёдный отказ каждой API с восстановлением | `python -X utf8 student-tests/test_recovery.py --mode failures` | Остановлены реальные api-1 и api-2, измерены RPS/ошибки/время, потерь и дублей 0 | [отказ 1](evidence/load-mtusi-lab3-osokin-api-1.json), [отказ 2](evidence/load-mtusi-lab3-osokin-api-2.json) |
|
||||
| CPU/RAM, приватные API, нет container_name | inspect/stats при проверках | Квоты из таблицы, API host port отсутствует, secrets/volumes по назначению | [processes.txt](evidence/processes.txt), `resources` в raw JSON |
|
||||
| Поочерёдная замена под нагрузкой, миграционные ограничения | `python -X utf8 student-tests/test_recovery.py --mode rolling` | Оба контейнера заменены, 1538 успешных операций, 66 ошибок, без потерь/дублей | [load-rolling.json](evidence/baseline/load-rolling.json), [rolling-processes.json](evidence/baseline/rolling-processes.json) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Ожидание: при остановке любого одного API сосед продолжит чтение/запись с прежними сессиями не позднее 10 s, повторы с исходным ключом не создадут дублей. Последовательная замена двух API сохранит корректность.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Среда: Windows 10, Docker Desktop/WSL2 Linux 6.18.54.1, x86_64; Docker видит 8 CPU и 8 275 038 208 bytes RAM (~7.71 GiB). Лимиты фиксированы в таблице выше. Генератор работает на том же ноутбуке.
|
||||
|
||||
Профиль: **4 параллельных клиента, closed loop**, каждый последовательно выполняет POST сообщения и GET первых 100 сообщений чата, затем ждёт 30 ms. Неопределённый POST повторяется с прежними key/body. Каждый сценарий около 32 s, первые 5 s до воздействия. Таблица включает всё окно с этими 5 s. При отказе API выключена 10 s после завершения stop, затем восстанавливается; следующая проверка начинается с двумя healthy API. Rolling заменяет обе исходные API по очереди. По одному прогону каждого сценария, одинаковый профиль; статистическая достоверность большого benchmark не заявляется.
|
||||
|
||||
В каждом сценарии новый чат с одним подтверждённым сообщением. Финальная история: 945 / 853 / 766 сообщений. После нагрузки проверяется вся история с пагинацией, все подтверждения и отсутствие дублей текстов/ID. Сохранённый до воздействия ключ повторяется после него и возвращает первоначальное сообщение. Каждый успешный GET во время нагрузки проверяет наличие начального сообщения.
|
||||
|
||||
| Воздействие | Окно, s | Операции / ошибки | Успешные RPS | Ошибки | p50 / p95 / p99 успешных, ms | Первый read / write после stop от начала воздействия, s |
|
||||
| --- | ---: | ---: | ---: | ---: | --- | --- |
|
||||
| Остановка api-1 | 32.253 | 1910 / 23 | 58.506 | 1.204% | 32.614 / 129.246 / 235.578 | 1.860 / 1.861 |
|
||||
| Остановка api-2 после восстановления api-1 | 32.316 | 1722 / 24 | 52.544 | 1.394% | 40.295 / 144.470 / 205.802 | 1.651 / 1.625 |
|
||||
| Поочерёдная замена обоих API | 32.262 | 1604 / 66 | 47.672 | 4.115% | 40.720 / 167.009 / 249.067 | Две последовательные замены; raw samples |
|
||||
|
||||
RPS = успешные GET+POST / время, не только сообщения/s. Перцентили рассчитаны по успешным операциям; тайминги ошибок сохранены в raw samples. Первый успех не означает конец переходных ошибок: последние ошибки после stop завершились через **1.963 s** и **1.620 s**. Успешные RPS в 8 s после stop: **52.250** и **37.125**. Подтверждённые данные потеряны: **0**, дубликаты: **0** во всех сценариях.
|
||||
|
||||
Выборки `docker stats --no-stream` собирались приблизительно раз в 5 s плюс время команды. Наблюдавшиеся максимумы по сценариям: API **104.83% CPU одного ядра / 144.9 MiB**, PostgreSQL **50.86% / 42.73 MiB**, HAProxy **26.55% / 13.22 MiB**, Redis **12.73% / 7.41 MiB**. Небольшое превышение 100% в короткой выборке CPU возможно при учёте квот/интервалов; фактическая квота API — 1 CPU. Это редкие наблюдения, не непрерывные максимумы; CPU и RAM не обязательно относятся к одному моменту. Полные выборки и запросы в raw JSON не содержат токенов/паролей.
|
||||
|
||||
| Воздействие | Ожидание | Наблюдение | Восстановление / данные |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Restart API + DB + Redis + LB | Данные и действующие сессии сохраняются | Public persistence PASS, volumes прежние | Старый токен работает, потерь 0 |
|
||||
| Отказ одной API | Сосед обслуживает без нового входа | Проверена каждая исходная API | read/write <2 s в обоих опытах, потерь 0 |
|
||||
| Одновременный повтор на двух API | Одна логическая запись | 16 одинаковых ответов, одна запись | Дубликатов 0 |
|
||||
| Последовательная замена | Данные и результаты повторов сохраняются | Новые контейнеры/PID, прежний ключ вернул исходное сообщение | 4.115% переходных ошибок, потерь/дублей 0 |
|
||||
|
||||
Разница RPS не доказывает масштабирование: отдельного baseline/серии повторов нет, история растёт, фоновая нагрузка ноутбука меняется. Исследование saturation point, серии 1/2/4 API, графики масштабирования и многоузловой failover уровня 5 не проводились.
|
||||
|
||||
## Повторные проверки перед публикацией
|
||||
|
||||
Основная таблица выше относится к первому полному прогону. Исходные перезаписанные файлы сохранены в `evidence/baseline/`; остальные исходные результаты остались в `evidence/`. Повторные проверки на том же стенде 02.11.2026 дали следующие результаты:
|
||||
|
||||
| Сценарий | Успешные RPS | Ошибки / операции | Потери / дубликаты | Raw result |
|
||||
| --- | ---: | ---: | --- | --- |
|
||||
| mtusi-lab3-osokin-api-3 | 58.005 | 73 / 1938 | 0 / 0 | [load-mtusi-lab3-osokin-api-3.json](evidence/load-mtusi-lab3-osokin-api-3.json) |
|
||||
| mtusi-lab3-osokin-api-4 | 61.424 | 50 / 2024 | 0 / 0 | [load-mtusi-lab3-osokin-api-4.json](evidence/load-mtusi-lab3-osokin-api-4.json) |
|
||||
| Поочерёдная замена API | 46.956 | 103 / 1620 | 0 / 0 | [load-rolling.json](evidence/load-rolling.json) |
|
||||
|
||||
Это повторение сценариев уровня 4, без исследования масштабирования или многоузлового отказа.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Проверен отказ одного API-процесса. Единственные LB, DB, Redis и Docker-host остаются едиными точками отказа; многоузловой HA нет.
|
||||
- Короткие ошибки обнаружения и замены измерены; бесшовное обновление без единой ошибки не заявляется. Неопределённое сообщение клиент повторяет с исходным ключом.
|
||||
- Нужен отдельный `labctl.py rolling`; обычный `up --build` может заменить обе API одновременно. Будущие версии/сложные миграции проверяет оператор; автоматического rollback нет.
|
||||
- Гарантия повторов ограничена 24 часами. Старые ключи требуют будущей периодической очистки.
|
||||
- HTTP, файловые secrets, общий DB-role и Redis без пароля — учебные упрощения localhost-стенда. Удалённая демонстрация требует TLS и отдельной настройки доступа.
|
||||
- Один ноутбук, малый набор данных, четыре closed-loop клиента, по одному короткому прогону и редкие CPU/RAM samples подтверждают эти сценарии уровня 4, не универсальную производительность.
|
||||
|
||||
## Источники
|
||||
|
||||
- Задание lab3, [контракт](contracts/README.md), [COURSE.md](COURSE.md), [CONTRIBUTING.md](CONTRIBUTING.md), публичные тесты skeleton.
|
||||
- Собственная lab2: исходники, миграция 001, Redis persistence, зависимости и bootstrap.
|
||||
- [Docker Compose services](https://docs.docker.com/reference/compose-file/services/) — scale, secrets, healthcheck, limits и stop_grace_period.
|
||||
- [HAProxy 3.0 manual](https://docs.haproxy.org/3.0/configuration.html) — server-template, resolvers, healthcheck и timeouts.
|
||||
- [PostgreSQL advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS), [psycopg transactions](https://www.psycopg.org/psycopg3/docs/basic/transactions.html).
|
||||
- [Redis persistence](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [Python HTTP server](https://docs.python.org/3.13/library/http.server.html) и стандартные hashlib/secrets/signal.
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
name: mtusi-lab3-osokin
|
||||
x-app: &app
|
||||
image: mtusi-messenger:lab3
|
||||
build: .
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SESSION_STORE: redis
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
CURSOR_KEY_FILE: /run/secrets/cursor_key
|
||||
secrets: [database_url, cache_url, cursor_key]
|
||||
mem_limit: 256m
|
||||
cpus: 1
|
||||
dns_opt: ['timeout:1', 'attempts:1']
|
||||
networks: [backend]
|
||||
services:
|
||||
db:
|
||||
image: postgres:17.6-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
secrets: [db_password]
|
||||
volumes: [pgdata:/var/lib/postgresql/data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 2
|
||||
networks: [backend]
|
||||
redis:
|
||||
image: redis:7.4.5-bookworm
|
||||
command: [redis-server, /usr/local/etc/redis/redis.conf]
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks: [backend]
|
||||
migrate:
|
||||
<<: *app
|
||||
command: [python, -m, messenger.migrate]
|
||||
healthcheck: {disable: true}
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
api:
|
||||
<<: *app
|
||||
scale: ${API_REPLICAS:-2}
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
start_period: 5s
|
||||
retries: 2
|
||||
lb:
|
||||
image: haproxy:3.0.16-alpine
|
||||
ports: ['127.0.0.1:${APP_PORT:-8080}:8080']
|
||||
volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro']
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
stop_grace_period: 10s
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks: [frontend, backend]
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
networks:
|
||||
frontend:
|
||||
backend:
|
||||
internal: true
|
||||
secrets:
|
||||
db_password: {file: ./secrets/db_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
cursor_key: {file: ./secrets/cursor_key}
|
||||
@@ -0,0 +1,4 @@
|
||||
# Исходный прогон
|
||||
|
||||
Результаты первого прогона 02.11.2026, приведённые в основной таблице REPORT.md.
|
||||
Сохранены отдельно, поскольку повторные запуски перезаписывают файлы в evidence/.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,216 @@
|
||||
{
|
||||
"before": [
|
||||
{
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 12364,
|
||||
"started_at": "2026-11-02T13:30:01.426804291Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 12419,
|
||||
"started_at": "2026-11-02T13:30:01.726545444Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "071b123e1f8c4e837ae997c8403efff92674321f2ccadef8714d4c8e38bc4ee8",
|
||||
"name": "mtusi-lab3-osokin-db-1",
|
||||
"service": "db",
|
||||
"number": 1,
|
||||
"hostname": "071b123e1f8c",
|
||||
"pid": 766,
|
||||
"started_at": "2026-11-02T13:25:24.897109157Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
|
||||
"memory_limit": 536870912,
|
||||
"cpu_limit": 2.0,
|
||||
"published_ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c30f4b49ac01fc1f6072e0a477f471e195c61bded9a5fc41c9f8ad3064f9022e",
|
||||
"name": "mtusi-lab3-osokin-lb-1",
|
||||
"service": "lb",
|
||||
"number": 1,
|
||||
"hostname": "c30f4b49ac01",
|
||||
"pid": 1375,
|
||||
"started_at": "2026-11-02T13:25:38.353420209Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "5184c3c05e75d5c59b414c876840215b913b830992f29e5e3b00f097b3af28d0",
|
||||
"name": "mtusi-lab3-osokin-redis-1",
|
||||
"service": "redis",
|
||||
"number": 1,
|
||||
"hostname": "5184c3c05e75",
|
||||
"pid": 696,
|
||||
"started_at": "2026-11-02T13:25:24.294225064Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 23269,
|
||||
"started_at": "2026-11-02T13:34:14.511777506Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 23303,
|
||||
"started_at": "2026-11-02T13:34:14.576732298Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "071b123e1f8c4e837ae997c8403efff92674321f2ccadef8714d4c8e38bc4ee8",
|
||||
"name": "mtusi-lab3-osokin-db-1",
|
||||
"service": "db",
|
||||
"number": 1,
|
||||
"hostname": "071b123e1f8c",
|
||||
"pid": 23212,
|
||||
"started_at": "2026-11-02T13:34:14.441800497Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
|
||||
"memory_limit": 536870912,
|
||||
"cpu_limit": 2.0,
|
||||
"published_ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c30f4b49ac01fc1f6072e0a477f471e195c61bded9a5fc41c9f8ad3064f9022e",
|
||||
"name": "mtusi-lab3-osokin-lb-1",
|
||||
"service": "lb",
|
||||
"number": 1,
|
||||
"hostname": "c30f4b49ac01",
|
||||
"pid": 23378,
|
||||
"started_at": "2026-11-02T13:34:14.739292592Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "5184c3c05e75d5c59b414c876840215b913b830992f29e5e3b00f097b3af28d0",
|
||||
"name": "mtusi-lab3-osokin-redis-1",
|
||||
"service": "redis",
|
||||
"number": 1,
|
||||
"hostname": "5184c3c05e75",
|
||||
"pid": 23247,
|
||||
"started_at": "2026-11-02T13:34:14.484264122Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"before": [
|
||||
{
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 25766,
|
||||
"started_at": "2026-11-02T13:35:13.89817575Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 27229,
|
||||
"started_at": "2026-11-02T13:35:53.034956399Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
],
|
||||
"replacements": [
|
||||
{
|
||||
"old": {
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 25766,
|
||||
"started_at": "2026-11-02T13:35:13.89817575Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 27229,
|
||||
"started_at": "2026-11-02T13:35:53.034956399Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "e77746bad0b9b0b63ea6e1f91df2108906611b4af7a98f19f734f1339b540e0d",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "e77746bad0b9",
|
||||
"pid": 28336,
|
||||
"started_at": "2026-11-02T13:36:21.512813258Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"old": {
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 27229,
|
||||
"started_at": "2026-11-02T13:35:53.034956399Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "e77746bad0b9b0b63ea6e1f91df2108906611b4af7a98f19f734f1339b540e0d",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "e77746bad0b9",
|
||||
"pid": 28336,
|
||||
"started_at": "2026-11-02T13:36:21.512813258Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "bace794949d524e5daf3b1feb390cee1ed39e13f5d793157b692d3b71559b8bf",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "bace794949d5",
|
||||
"pid": 28914,
|
||||
"started_at": "2026-11-02T13:36:35.917178941Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"id": "e77746bad0b9b0b63ea6e1f91df2108906611b4af7a98f19f734f1339b540e0d",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "e77746bad0b9",
|
||||
"pid": 28336,
|
||||
"started_at": "2026-11-02T13:36:21.512813258Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "bace794949d524e5daf3b1feb390cee1ed39e13f5d793157b692d3b71559b8bf",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "bace794949d5",
|
||||
"pid": 28914,
|
||||
"started_at": "2026-11-02T13:36:35.917178941Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"target_instance": "0700f348645b-f5a34903e39b",
|
||||
"stopped": {
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 23269,
|
||||
"started_at": "2026-11-02T13:34:14.511777506Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "0700f348645b5bd3a8b3f4a68230f6fb02ad423271cc7397039d3adae960252f",
|
||||
"name": "mtusi-lab3-osokin-api-1",
|
||||
"service": "api",
|
||||
"number": 1,
|
||||
"hostname": "0700f348645b",
|
||||
"pid": 0,
|
||||
"started_at": "2026-11-02T13:34:14.511777506Z",
|
||||
"status": "exited",
|
||||
"health": "unhealthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "e91ca53a67f8494106c6d014999693c2cbb3f4ec79db94d9a019668fdb24634a",
|
||||
"name": "mtusi-lab3-osokin-api-2",
|
||||
"service": "api",
|
||||
"number": 2,
|
||||
"hostname": "e91ca53a67f8",
|
||||
"pid": 23303,
|
||||
"started_at": "2026-11-02T13:34:14.576732298Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
],
|
||||
"hook_seconds": 0.9716456001624465
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,198 @@
|
||||
Cloning into 'C:\Users\blobb\OneDrive\Документи\ChatGPT\высоконагружноут\lab3\output\clean-check'...
|
||||
Local .env and secrets ready (existing values preserved).
|
||||
Image mtusi-messenger:lab3 Building
|
||||
Image mtusi-messenger:lab3 Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.32kB 0.1s done
|
||||
#1 DONE 0.1s
|
||||
|
||||
#2 [migrate internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 723B 0.0s done
|
||||
#2 DONE 0.1s
|
||||
|
||||
#3 [api internal] load metadata for docker.io/library/python:3.13.7-slim-bookworm
|
||||
#3 DONE 1.3s
|
||||
|
||||
#4 [api internal] load .dockerignore
|
||||
#4 transferring context: 224B 0.0s done
|
||||
#4 DONE 0.1s
|
||||
|
||||
#5 [migrate 1/11] FROM docker.io/library/python:3.13.7-slim-bookworm@sha256:adafcc17694d715c905b4c7bebd96907a1fd5cf183395f0ebc4d3428bd22d92d
|
||||
#5 resolve docker.io/library/python:3.13.7-slim-bookworm@sha256:adafcc17694d715c905b4c7bebd96907a1fd5cf183395f0ebc4d3428bd22d92d 0.1s done
|
||||
#5 DONE 0.1s
|
||||
|
||||
#6 [api internal] load build context
|
||||
#6 transferring context: 214.97kB 0.2s done
|
||||
#6 DONE 0.3s
|
||||
|
||||
#7 [api 2/11] WORKDIR /app
|
||||
#7 CACHED
|
||||
|
||||
#8 [api 3/11] COPY requirements.txt .
|
||||
#8 CACHED
|
||||
|
||||
#9 [api 4/11] RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
|
||||
#9 CACHED
|
||||
|
||||
#7 [migrate 2/11] WORKDIR /app
|
||||
#7 CACHED
|
||||
|
||||
#8 [migrate 3/11] COPY requirements.txt .
|
||||
#8 CACHED
|
||||
|
||||
#9 [migrate 4/11] RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app
|
||||
#9 CACHED
|
||||
|
||||
#10 [migrate 5/11] COPY src ./src
|
||||
#10 DONE 0.1s
|
||||
|
||||
#11 [migrate 6/11] COPY migrations ./migrations
|
||||
#11 DONE 0.1s
|
||||
|
||||
#12 [migrate 7/11] COPY contracts ./contracts
|
||||
#12 DONE 0.1s
|
||||
|
||||
#13 [api 8/11] COPY tests ./tests
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 [migrate 9/11] COPY student-tests ./student-tests
|
||||
#14 DONE 0.1s
|
||||
|
||||
#15 [api 10/11] COPY scripts ./scripts
|
||||
#15 DONE 0.1s
|
||||
|
||||
#16 [migrate 11/11] COPY lab.json requirements-dev.txt ./
|
||||
#16 DONE 0.1s
|
||||
|
||||
#17 [migrate] exporting to image
|
||||
#17 exporting layers
|
||||
#17 exporting layers 0.9s done
|
||||
#17 exporting manifest sha256:3beed57d1d3b67ec2fa74e6e9f6d13f6da86c52be76c15dcd2030881cb9bbdfb 0.1s done
|
||||
#17 exporting config sha256:fc0421d9ce68e7a2ca3e540ea36a1aca4df51c6042d6c05876867fa4291b17f3
|
||||
#17 exporting config sha256:fc0421d9ce68e7a2ca3e540ea36a1aca4df51c6042d6c05876867fa4291b17f3 0.1s done
|
||||
#17 exporting attestation manifest sha256:65e1bb3a9940d30bccf64bde2e9f573ec47670040de09ed6f38f9448e4db0c7e
|
||||
#17 exporting attestation manifest sha256:65e1bb3a9940d30bccf64bde2e9f573ec47670040de09ed6f38f9448e4db0c7e 0.1s done
|
||||
#17 exporting manifest list sha256:a92127e2f508e2d482561d8fffe63c9555365d992c82e22f999751cb19ef3b8f 0.0s done
|
||||
#17 naming to docker.io/library/mtusi-messenger:lab3
|
||||
#17 naming to docker.io/library/mtusi-messenger:lab3 0.0s done
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab3
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab3 0.8s done
|
||||
#17 DONE 2.1s
|
||||
|
||||
#18 [api] exporting to image
|
||||
#18 exporting layers 0.9s done
|
||||
#18 exporting manifest sha256:b8cd4b1a86d1cd3715a6cbfdf1c34ea16a0f4389abc389f36d1adfe2d642f099 0.1s done
|
||||
#18 exporting config sha256:1648fa82582d101839573d78abf6dab6de8882b316b398f6c04e7a36cf9c3605 0.1s done
|
||||
#18 exporting attestation manifest sha256:4f1e8c8f70eaccc53915c6c217a0ab1df09129141451ee64ede06ea52c9b7e29 0.1s done
|
||||
#18 exporting manifest list sha256:2366ffcea5e3af49173d51e00d561f12a1f894b02fea72318d4d379a766f77a2 0.0s done
|
||||
#18 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#18 unpacking to docker.io/library/mtusi-messenger:lab3 0.8s done
|
||||
#18 DONE 2.3s
|
||||
|
||||
#19 [migrate] resolving provenance for metadata file
|
||||
#19 DONE 0.0s
|
||||
|
||||
#20 [api] resolving provenance for metadata file
|
||||
#20 DONE 0.0s
|
||||
Image mtusi-messenger:lab3 Built
|
||||
Image mtusi-messenger:lab3 Built
|
||||
Network mtusi-lab3-osokin-clean_backend Creating
|
||||
Network mtusi-lab3-osokin-clean_frontend Creating
|
||||
Network mtusi-lab3-osokin-clean_frontend Creating
|
||||
Volume mtusi-lab3-osokin-clean_pgdata Creating
|
||||
Volume mtusi-lab3-osokin-clean_pgdata Creating
|
||||
Network mtusi-lab3-osokin-clean_backend Creating
|
||||
Volume mtusi-lab3-osokin-clean_redisdata Creating
|
||||
Volume mtusi-lab3-osokin-clean_redisdata Creating
|
||||
Volume mtusi-lab3-osokin-clean_pgdata Created
|
||||
Volume mtusi-lab3-osokin-clean_pgdata Created
|
||||
Volume mtusi-lab3-osokin-clean_redisdata Created
|
||||
Volume mtusi-lab3-osokin-clean_redisdata Created
|
||||
Network mtusi-lab3-osokin-clean_frontend Created
|
||||
Network mtusi-lab3-osokin-clean_frontend Created
|
||||
Network mtusi-lab3-osokin-clean_backend Created
|
||||
Network mtusi-lab3-osokin-clean_backend Created
|
||||
Container mtusi-lab3-osokin-clean-db-1 Creating
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Creating
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Created
|
||||
Container mtusi-lab3-osokin-clean-db-1 Created
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Creating
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Created
|
||||
Container mtusi-lab3-osokin-clean-api-2 Creating
|
||||
Container mtusi-lab3-osokin-clean-api-1 Creating
|
||||
Container mtusi-lab3-osokin-clean-api-1 Created
|
||||
Container mtusi-lab3-osokin-clean-api-2 Created
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Creating
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Created
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Starting
|
||||
Container mtusi-lab3-osokin-clean-db-1 Starting
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Started
|
||||
Container mtusi-lab3-osokin-clean-db-1 Started
|
||||
Container mtusi-lab3-osokin-clean-db-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-db-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Starting
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Started
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Exited
|
||||
Container mtusi-lab3-osokin-clean-api-2 Starting
|
||||
Container mtusi-lab3-osokin-clean-api-2 Started
|
||||
Container mtusi-lab3-osokin-clean-api-1 Starting
|
||||
Container mtusi-lab3-osokin-clean-api-1 Started
|
||||
Container mtusi-lab3-osokin-clean-api-2 Waiting
|
||||
Container mtusi-lab3-osokin-clean-api-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-api-2 Healthy
|
||||
Container mtusi-lab3-osokin-clean-api-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Starting
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Started
|
||||
Container mtusi-lab3-osokin-clean-api-2 Waiting
|
||||
Container mtusi-lab3-osokin-clean-api-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-db-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Waiting
|
||||
Container mtusi-lab3-osokin-clean-db-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Exited
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-api-2 Healthy
|
||||
Container mtusi-lab3-osokin-clean-api-1 Healthy
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Healthy
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
{"result": "PASS", "parallel_requests": 16, "api_instances": ["9ac6846ff6f9-a8e7510753d6", "c4dd0a2e02eb-13ddbadaf451"], "unique_messages": 1, "scope_conflict_logout": "PASS"}
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab3-osokin-clean-api-1 mtusi-messenger:lab3 "python -m messenger…" api 22 seconds ago Up 13 seconds (healthy) 8080/tcp
|
||||
mtusi-lab3-osokin-clean-api-2 mtusi-messenger:lab3 "python -m messenger…" api 22 seconds ago Up 13 seconds (healthy) 8080/tcp
|
||||
mtusi-lab3-osokin-clean-db-1 postgres:17.6-bookworm "docker-entrypoint.s…" db 23 seconds ago Up 21 seconds (healthy) 5432/tcp
|
||||
mtusi-lab3-osokin-clean-lb-1 haproxy:3.0.16-alpine "docker-entrypoint.s…" lb 22 seconds ago Up 7 seconds (healthy) 127.0.0.1:8081->8080/tcp
|
||||
mtusi-lab3-osokin-clean-migrate-1 mtusi-messenger:lab3 "python -m messenger…" migrate 23 seconds ago Exited (0) 14 seconds ago
|
||||
mtusi-lab3-osokin-clean-redis-1 redis:7.4.5-bookworm "docker-entrypoint.s…" redis 23 seconds ago Up 21 seconds (healthy) 6379/tcp
|
||||
PASS: committed clean clone, new secrets, fresh project volumes, smoke and concurrent replay.
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Stopping
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Stopped
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Removing
|
||||
Container mtusi-lab3-osokin-clean-lb-1 Removed
|
||||
Container mtusi-lab3-osokin-clean-api-1 Stopping
|
||||
Container mtusi-lab3-osokin-clean-api-2 Stopping
|
||||
Container mtusi-lab3-osokin-clean-api-1 Stopped
|
||||
Container mtusi-lab3-osokin-clean-api-1 Removing
|
||||
Container mtusi-lab3-osokin-clean-api-2 Stopped
|
||||
Container mtusi-lab3-osokin-clean-api-2 Removing
|
||||
Container mtusi-lab3-osokin-clean-api-1 Removed
|
||||
Container mtusi-lab3-osokin-clean-api-2 Removed
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Stopping
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Stopping
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Stopped
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Removing
|
||||
Container mtusi-lab3-osokin-clean-migrate-1 Removed
|
||||
Container mtusi-lab3-osokin-clean-db-1 Stopping
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Stopped
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Removing
|
||||
Container mtusi-lab3-osokin-clean-redis-1 Removed
|
||||
Container mtusi-lab3-osokin-clean-db-1 Stopped
|
||||
Container mtusi-lab3-osokin-clean-db-1 Removing
|
||||
Container mtusi-lab3-osokin-clean-db-1 Removed
|
||||
Network mtusi-lab3-osokin-clean_backend Removing
|
||||
Network mtusi-lab3-osokin-clean_frontend Removing
|
||||
Network mtusi-lab3-osokin-clean_backend Removed
|
||||
Network mtusi-lab3-osokin-clean_frontend Removed
|
||||
@@ -0,0 +1 @@
|
||||
$ docker compose --env-file .env config --quiet
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"verified_at_utc": "2026-11-02T13:36:55.726875+00:00",
|
||||
"host_os": "Windows-10-10.0.19045-SP0",
|
||||
"client_python": "3.14.3",
|
||||
"docker": {
|
||||
"ServerVersion": "29.8.2",
|
||||
"OperatingSystem": "Docker Desktop",
|
||||
"KernelVersion": "6.18.54.1-microsoft-standard-WSL2",
|
||||
"NCPU": 8,
|
||||
"MemTotal": 8275038208,
|
||||
"Architecture": "x86_64"
|
||||
},
|
||||
"images": {
|
||||
"postgres:17.6-bookworm": [
|
||||
"postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3"
|
||||
],
|
||||
"redis:7.4.5-bookworm": [
|
||||
"redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb"
|
||||
],
|
||||
"haproxy:3.0.16-alpine": [
|
||||
"haproxy@sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
$ python tests/resilience.py --mode failover --base-url http://127.0.0.1:8080 --action-script scripts/stop-one.cmd
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 2, "first_recovery_seconds": 1.852}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
$ docker compose exec -T api python student-tests/test_idempotency.py
|
||||
{"result": "PASS", "parallel_requests": 16, "api_instances": ["0700f348645b-b7e352b8d6c8", "e91ca53a67f8-3f2d443c13b7"], "unique_messages": 1, "scope_conflict_logout": "PASS"}
|
||||
@@ -0,0 +1,24 @@
|
||||
$ python student-tests/test_recovery.py
|
||||
Container mtusi-lab3-osokin-api-2 Running
|
||||
Container mtusi-lab3-osokin-api-1 Starting
|
||||
Container mtusi-lab3-osokin-api-1 Started
|
||||
{"mode": "stop", "target": "mtusi-lab3-osokin-api-1", "at_utc": "2026-11-02T13:35:33.921725+00:00", "duration_seconds": 32.253, "warmup_seconds": 5, "concurrency": 4, "profile": "each worker: POST + GET, then 30ms pause; closed loop", "initial_messages": 1, "final_messages": 945, "confirmations": 944, "operations": 1910, "successful_operations": 1887, "errors": 23, "error_percent": 1.204, "successful_rps": 58.506, "latency_success_ms": {"p50": 32.614, "p95": 129.246, "p99": 235.578}, "instances_before": ["0700f348645b-8f809d1e50c5", "e91ca53a67f8-c1d5fe30dae7"], "confirmed_data_lost": 0, "duplicate_messages": 0, "first_success_after_stop_seconds_from_action_start": {"read": 1.86, "write": 1.861}, "last_error_seconds_from_action_start": 1.963, "successful_rps_8s_after_stop": 52.25}
|
||||
Evidence: evidence/load-mtusi-lab3-osokin-api-1.json
|
||||
Container mtusi-lab3-osokin-api-1 Running
|
||||
Container mtusi-lab3-osokin-api-2 Starting
|
||||
Container mtusi-lab3-osokin-api-2 Started
|
||||
{"mode": "stop", "target": "mtusi-lab3-osokin-api-2", "at_utc": "2026-11-02T13:36:11.301910+00:00", "duration_seconds": 32.316, "warmup_seconds": 5, "concurrency": 4, "profile": "each worker: POST + GET, then 30ms pause; closed loop", "initial_messages": 1, "final_messages": 853, "confirmations": 852, "operations": 1722, "successful_operations": 1698, "errors": 24, "error_percent": 1.394, "successful_rps": 52.544, "latency_success_ms": {"p50": 40.295, "p95": 144.47, "p99": 205.802}, "instances_before": ["0700f348645b-1fa55a5631e0", "e91ca53a67f8-c1d5fe30dae7"], "confirmed_data_lost": 0, "duplicate_messages": 0, "first_success_after_stop_seconds_from_action_start": {"read": 1.651, "write": 1.625}, "last_error_seconds_from_action_start": 1.62, "successful_rps_8s_after_stop": 37.125}
|
||||
Evidence: evidence/load-mtusi-lab3-osokin-api-2.json
|
||||
Container mtusi-lab3-osokin-api-2 Running
|
||||
Container mtusi-lab3-osokin-api-3 Creating
|
||||
Container mtusi-lab3-osokin-api-3 Created
|
||||
Container mtusi-lab3-osokin-api-3 Starting
|
||||
Container mtusi-lab3-osokin-api-3 Started
|
||||
Container mtusi-lab3-osokin-api-3 Running
|
||||
Container mtusi-lab3-osokin-api-4 Creating
|
||||
Container mtusi-lab3-osokin-api-4 Created
|
||||
Container mtusi-lab3-osokin-api-4 Starting
|
||||
Container mtusi-lab3-osokin-api-4 Started
|
||||
{"mode": "rolling", "target": null, "at_utc": "2026-11-02T13:36:51.491875+00:00", "duration_seconds": 32.262, "warmup_seconds": 5, "concurrency": 4, "profile": "each worker: POST + GET, then 30ms pause; closed loop", "initial_messages": 1, "final_messages": 766, "confirmations": 765, "operations": 1604, "successful_operations": 1538, "errors": 66, "error_percent": 4.115, "successful_rps": 47.672, "latency_success_ms": {"p50": 40.72, "p95": 167.009, "p99": 249.067}, "instances_before": ["0700f348645b-1fa55a5631e0", "e91ca53a67f8-bfb725dd79c4"], "confirmed_data_lost": 0, "duplicate_messages": 0}
|
||||
Evidence: evidence/load-rolling.json
|
||||
PASS: grade 4 load scenarios; no scale or multi-node research track.
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
$ docker compose run --rm --no-deps migrate
|
||||
Container mtusi-lab3-osokin-migrate-run-edf517c23d72 Creating
|
||||
Container mtusi-lab3-osokin-migrate-run-edf517c23d72 Created
|
||||
Already applied: 001_initial.sql
|
||||
Already applied: 002_idempotency.sql
|
||||
Migrations committed.
|
||||
@@ -0,0 +1,3 @@
|
||||
$ python tests/resilience.py --mode persistence --base-url http://127.0.0.1:8080 --action-script scripts/restart.cmd
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,124 @@
|
||||
$ python scripts/labctl.py state
|
||||
[
|
||||
{
|
||||
"id": "e77746bad0b9b0b63ea6e1f91df2108906611b4af7a98f19f734f1339b540e0d",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "e77746bad0b9",
|
||||
"pid": 28336,
|
||||
"started_at": "2026-11-02T13:36:21.512813258Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "bace794949d524e5daf3b1feb390cee1ed39e13f5d793157b692d3b71559b8bf",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "bace794949d5",
|
||||
"pid": 28914,
|
||||
"started_at": "2026-11-02T13:36:35.917178941Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "071b123e1f8c4e837ae997c8403efff92674321f2ccadef8714d4c8e38bc4ee8",
|
||||
"name": "mtusi-lab3-osokin-db-1",
|
||||
"service": "db",
|
||||
"number": 1,
|
||||
"hostname": "071b123e1f8c",
|
||||
"pid": 23212,
|
||||
"started_at": "2026-11-02T13:34:14.441800497Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
|
||||
"memory_limit": 536870912,
|
||||
"cpu_limit": 2.0,
|
||||
"published_ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c30f4b49ac01fc1f6072e0a477f471e195c61bded9a5fc41c9f8ad3064f9022e",
|
||||
"name": "mtusi-lab3-osokin-lb-1",
|
||||
"service": "lb",
|
||||
"number": 1,
|
||||
"hostname": "c30f4b49ac01",
|
||||
"pid": 23378,
|
||||
"started_at": "2026-11-02T13:34:14.739292592Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "82311c8061080b390b484b4bd841b2cdae56c5b50d1972fa2923fe9311c8f7fa",
|
||||
"name": "mtusi-lab3-osokin-migrate-1",
|
||||
"service": "migrate",
|
||||
"number": 1,
|
||||
"hostname": "82311c806108",
|
||||
"pid": 0,
|
||||
"started_at": "2026-11-02T13:29:59.621073044Z",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"image": "sha256:0e84db5e71b3243fb1788ca5681dac9ed806ce4ffcb80fd4fa59672ecb4668b3",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "5184c3c05e75d5c59b414c876840215b913b830992f29e5e3b00f097b3af28d0",
|
||||
"name": "mtusi-lab3-osokin-redis-1",
|
||||
"service": "redis",
|
||||
"number": 1,
|
||||
"hostname": "5184c3c05e75",
|
||||
"pid": 23247,
|
||||
"started_at": "2026-11-02T13:34:14.484264122Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,216 @@
|
||||
{
|
||||
"before": [
|
||||
{
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 36331,
|
||||
"started_at": "2026-11-02T17:35:37.246585773Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 37833,
|
||||
"started_at": "2026-11-02T17:36:16.924518378Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "071b123e1f8c4e837ae997c8403efff92674321f2ccadef8714d4c8e38bc4ee8",
|
||||
"name": "mtusi-lab3-osokin-db-1",
|
||||
"service": "db",
|
||||
"number": 1,
|
||||
"hostname": "071b123e1f8c",
|
||||
"pid": 23212,
|
||||
"started_at": "2026-11-02T13:34:14.441800497Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
|
||||
"memory_limit": 536870912,
|
||||
"cpu_limit": 2.0,
|
||||
"published_ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c30f4b49ac01fc1f6072e0a477f471e195c61bded9a5fc41c9f8ad3064f9022e",
|
||||
"name": "mtusi-lab3-osokin-lb-1",
|
||||
"service": "lb",
|
||||
"number": 1,
|
||||
"hostname": "c30f4b49ac01",
|
||||
"pid": 23378,
|
||||
"started_at": "2026-11-02T13:34:14.739292592Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "5184c3c05e75d5c59b414c876840215b913b830992f29e5e3b00f097b3af28d0",
|
||||
"name": "mtusi-lab3-osokin-redis-1",
|
||||
"service": "redis",
|
||||
"number": 1,
|
||||
"hostname": "5184c3c05e75",
|
||||
"pid": 23247,
|
||||
"started_at": "2026-11-02T13:34:14.484264122Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 39964,
|
||||
"started_at": "2026-11-02T17:37:03.681246278Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 40009,
|
||||
"started_at": "2026-11-02T17:37:03.736590611Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "071b123e1f8c4e837ae997c8403efff92674321f2ccadef8714d4c8e38bc4ee8",
|
||||
"name": "mtusi-lab3-osokin-db-1",
|
||||
"service": "db",
|
||||
"number": 1,
|
||||
"hostname": "071b123e1f8c",
|
||||
"pid": 39911,
|
||||
"started_at": "2026-11-02T17:37:03.571308856Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3",
|
||||
"memory_limit": 536870912,
|
||||
"cpu_limit": 2.0,
|
||||
"published_ports": {
|
||||
"5432/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_pgdata",
|
||||
"destination": "/var/lib/postgresql/data"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "c30f4b49ac01fc1f6072e0a477f471e195c61bded9a5fc41c9f8ad3064f9022e",
|
||||
"name": "mtusi-lab3-osokin-lb-1",
|
||||
"service": "lb",
|
||||
"number": 1,
|
||||
"hostname": "c30f4b49ac01",
|
||||
"pid": 40127,
|
||||
"started_at": "2026-11-02T17:37:04.129257642Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:0e310ea3294f7854692c37898920fd2c04ea6abdb89724092e93280a2b85b4cf",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "5184c3c05e75d5c59b414c876840215b913b830992f29e5e3b00f097b3af28d0",
|
||||
"name": "mtusi-lab3-osokin-redis-1",
|
||||
"service": "redis",
|
||||
"number": 1,
|
||||
"hostname": "5184c3c05e75",
|
||||
"pid": 39869,
|
||||
"started_at": "2026-11-02T17:37:03.412252585Z",
|
||||
"status": "running",
|
||||
"health": "starting",
|
||||
"image": "sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb",
|
||||
"memory_limit": 134217728,
|
||||
"cpu_limit": 0.5,
|
||||
"published_ports": {
|
||||
"6379/tcp": null
|
||||
},
|
||||
"mounts": [
|
||||
{
|
||||
"name": "mtusi-lab3-osokin_redisdata",
|
||||
"destination": "/data"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
$ python scripts/labctl.py restore
|
||||
Container mtusi-lab3-osokin-api-2 Running
|
||||
Container mtusi-lab3-osokin-api-1 Starting
|
||||
Container mtusi-lab3-osokin-api-1 Started
|
||||
@@ -0,0 +1,3 @@
|
||||
$ python scripts/labctl.py restore
|
||||
Container mtusi-lab3-osokin-api-1 Running
|
||||
Container mtusi-lab3-osokin-api-2 Running
|
||||
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"before": [
|
||||
{
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 39964,
|
||||
"started_at": "2026-11-02T17:37:03.681246278Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 40009,
|
||||
"started_at": "2026-11-02T17:37:03.736590611Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
],
|
||||
"replacements": [
|
||||
{
|
||||
"old": {
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 39964,
|
||||
"started_at": "2026-11-02T17:37:03.681246278Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 40009,
|
||||
"started_at": "2026-11-02T17:37:03.736590611Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "43b4dbfa36eb4a10834498e26b20b2860e942f833cd44d1572d299415539a7cf",
|
||||
"name": "mtusi-lab3-osokin-api-5",
|
||||
"service": "api",
|
||||
"number": 5,
|
||||
"hostname": "43b4dbfa36eb",
|
||||
"pid": 42435,
|
||||
"started_at": "2026-11-02T17:37:57.758591108Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"old": {
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 40009,
|
||||
"started_at": "2026-11-02T17:37:03.736590611Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "43b4dbfa36eb4a10834498e26b20b2860e942f833cd44d1572d299415539a7cf",
|
||||
"name": "mtusi-lab3-osokin-api-5",
|
||||
"service": "api",
|
||||
"number": 5,
|
||||
"hostname": "43b4dbfa36eb",
|
||||
"pid": 42435,
|
||||
"started_at": "2026-11-02T17:37:57.758591108Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "dcf5930418702b4ecc629d16661e4f616d7c2a4caa5290517d203fa7a493e728",
|
||||
"name": "mtusi-lab3-osokin-api-6",
|
||||
"service": "api",
|
||||
"number": 6,
|
||||
"hostname": "dcf593041870",
|
||||
"pid": 42989,
|
||||
"started_at": "2026-11-02T17:38:10.934268194Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"id": "43b4dbfa36eb4a10834498e26b20b2860e942f833cd44d1572d299415539a7cf",
|
||||
"name": "mtusi-lab3-osokin-api-5",
|
||||
"service": "api",
|
||||
"number": 5,
|
||||
"hostname": "43b4dbfa36eb",
|
||||
"pid": 42435,
|
||||
"started_at": "2026-11-02T17:37:57.758591108Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "dcf5930418702b4ecc629d16661e4f616d7c2a4caa5290517d203fa7a493e728",
|
||||
"name": "mtusi-lab3-osokin-api-6",
|
||||
"service": "api",
|
||||
"number": 6,
|
||||
"hostname": "dcf593041870",
|
||||
"pid": 42989,
|
||||
"started_at": "2026-11-02T17:38:10.934268194Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,2 @@
|
||||
$ python tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"target_instance": "32ed9304362f-4dd4aebce562",
|
||||
"stopped": {
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 27569,
|
||||
"started_at": "2026-11-02T17:31:47.728388242Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
},
|
||||
"after": [
|
||||
{
|
||||
"id": "32ed9304362f418c78806b66482ea7b642b83fff81dbed6f33b66174f5c14a59",
|
||||
"name": "mtusi-lab3-osokin-api-3",
|
||||
"service": "api",
|
||||
"number": 3,
|
||||
"hostname": "32ed9304362f",
|
||||
"pid": 0,
|
||||
"started_at": "2026-11-02T17:31:47.728388242Z",
|
||||
"status": "exited",
|
||||
"health": "unhealthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {},
|
||||
"mounts": []
|
||||
},
|
||||
{
|
||||
"id": "6a9d2dc65303943669653bdaa5c660593d0e8760c1ccadd2fbad2d0571112a7a",
|
||||
"name": "mtusi-lab3-osokin-api-4",
|
||||
"service": "api",
|
||||
"number": 4,
|
||||
"hostname": "6a9d2dc65303",
|
||||
"pid": 27645,
|
||||
"started_at": "2026-11-02T17:31:48.008601983Z",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"image": "sha256:60edae93ffe5cfb7aa028391005ca54aaa1b3f5ca75ae2db56820930297bc593",
|
||||
"memory_limit": 268435456,
|
||||
"cpu_limit": 1.0,
|
||||
"published_ports": {
|
||||
"8080/tcp": null
|
||||
},
|
||||
"mounts": []
|
||||
}
|
||||
],
|
||||
"hook_seconds": 1.0964262997731566
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
global
|
||||
maxconn 256
|
||||
log stdout format raw local0 warning
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
timeout connect 500ms
|
||||
timeout client 8s
|
||||
timeout server 8s
|
||||
timeout http-request 3s
|
||||
timeout queue 1s
|
||||
timeout check 1s
|
||||
retries 0
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold refused 1s
|
||||
hold other 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend messenger
|
||||
|
||||
backend messenger
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 1s downinter 1s fall 2 rise 1
|
||||
server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr none
|
||||
@@ -0,0 +1,10 @@
|
||||
bind 0.0.0.0
|
||||
protected-mode no
|
||||
port 6379
|
||||
dir /data
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
save ""
|
||||
maxmemory 64mb
|
||||
maxmemory-policy noeviction
|
||||
# Only the private Compose network can reach Redis; no host port is published.
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
UNIQUE (chat_id, position)
|
||||
);
|
||||
CREATE INDEX chat_members_user ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_order ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history ON messages(chat_id, created_at, id);
|
||||
@@ -0,0 +1,12 @@
|
||||
-- Additive migration: old lab2 tables and rows are unchanged.
|
||||
-- This table is only for POST /chats/{chat_id}/messages; method/path scope is implicit.
|
||||
CREATE TABLE message_requests (
|
||||
actor_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
fingerprint text NOT NULL,
|
||||
response jsonb NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (actor_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX message_requests_expiry ON message_requests(expires_at);
|
||||
@@ -0,0 +1,5 @@
|
||||
psycopg==3.2.10
|
||||
psycopg-binary==3.2.10
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.4.0
|
||||
typing_extensions==4.15.0
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials once; never print them or replace existing files."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(exist_ok=True)
|
||||
password_file = folder / 'db_password'
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(24), encoding='utf-8')
|
||||
password = password_file.read_text(encoding='utf-8').strip()
|
||||
values = {
|
||||
'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
'cursor_key': secrets.token_hex(32),
|
||||
}
|
||||
for name, value in values.items():
|
||||
target = folder / name
|
||||
if not target.exists():
|
||||
target.write_text(value, encoding='utf-8')
|
||||
if not (root / '.env').exists():
|
||||
(root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8080\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\n', encoding='utf-8')
|
||||
print('Local .env and secrets ready (existing values preserved).')
|
||||
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build committed sources with fresh secrets/volumes on a separate local test port."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CLONE = ROOT / 'output' / 'clean-check'
|
||||
ENV = dict(os.environ, PYTHONUTF8='1', PYTHONIOENCODING='utf-8', APP_PORT='8081', API_REPLICAS='2')
|
||||
|
||||
|
||||
def run(args, cwd=CLONE):
|
||||
result = subprocess.run(args, cwd=cwd, env=ENV, text=True, encoding='utf-8', errors='replace',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
print(result.stdout, end='', flush=True)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Clean checkout check failed; see command output')
|
||||
|
||||
|
||||
def main():
|
||||
if CLONE.exists():
|
||||
raise SystemExit('output/clean-check already exists; preserve it or move it before a fresh run')
|
||||
CLONE.parent.mkdir(exist_ok=True)
|
||||
run(['git', 'clone', '--no-hardlinks', str(ROOT), str(CLONE)], cwd=ROOT)
|
||||
assert not (CLONE / '.env').exists() and not (CLONE / 'secrets').exists()
|
||||
run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py'])
|
||||
dc = ['docker', 'compose', '-p', 'mtusi-lab3-osokin-clean', '--env-file', '.env']
|
||||
try:
|
||||
run([*dc, 'up', '-d', '--build', '--wait'])
|
||||
run([sys.executable, '-X', 'utf8', 'tests/smoke.py', '--base-url', 'http://127.0.0.1:8081'])
|
||||
run([*dc, 'exec', '-T', 'api', 'python', 'student-tests/test_idempotency.py'])
|
||||
run([*dc, 'ps', '--all'])
|
||||
print('PASS: committed clean clone, new secrets, fresh project volumes, smoke and concurrent replay.')
|
||||
finally:
|
||||
run([*dc, 'down']) # Keep even the disposable test data; never down --volumes here.
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Short defense demonstration, with tokens and credentials kept only in memory."""
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
c = Client(os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
instances = Counter(c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
for _ in range(12))
|
||||
check(len(instances) == 2, 'Two API instances must serve the same session')
|
||||
print('One login; 12 requests through LB:')
|
||||
for instance, count in sorted(instances.items()):
|
||||
print(' ', instance, ':', count, 'requests')
|
||||
key = uuid.uuid4().hex
|
||||
first = c.message(alice, chat, 'Одно сообщение при двух повторах', key=key)
|
||||
second = c.message(alice, chat, 'Одно сообщение при двух повторах', key=key)
|
||||
check(first['json'] == second['json'], 'Replay differs')
|
||||
print('POST #1:', first['status'], first['headers']['X-Instance-Id'])
|
||||
print('POST #2:', second['status'], second['headers']['X-Instance-Id'])
|
||||
print('Same message ID:', first['json']['id'])
|
||||
print('PASS: shared session, different processes, one message.')
|
||||
@@ -0,0 +1,136 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Operate only this Compose project; never remove a data volume or log secrets."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8').strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command('docker', 'compose', '--env-file', '.env', *args)
|
||||
|
||||
|
||||
def containers(service=None, all_states=False):
|
||||
args = ['ps', '-q'] + (['--all'] if all_states else []) + ([service] if service else [])
|
||||
ids = compose(*args).split()
|
||||
if not ids:
|
||||
return []
|
||||
raw = json.loads(command('docker', 'inspect', *ids))
|
||||
result = []
|
||||
for item in raw:
|
||||
labels = item['Config']['Labels']
|
||||
result.append(dict(id=item['Id'], name=item['Name'].lstrip('/'),
|
||||
service=labels['com.docker.compose.service'],
|
||||
number=int(labels.get('com.docker.compose.container-number', '1')),
|
||||
hostname=item['Config']['Hostname'], pid=item['State']['Pid'],
|
||||
started_at=item['State']['StartedAt'], status=item['State']['Status'],
|
||||
health=item['State'].get('Health', {}).get('Status'),
|
||||
image=item['Image'],
|
||||
memory_limit=item['HostConfig']['Memory'],
|
||||
cpu_limit=item['HostConfig']['NanoCpus'] / 1e9,
|
||||
published_ports=item['NetworkSettings']['Ports'],
|
||||
mounts=[dict(name=m.get('Name'), destination=m['Destination'])
|
||||
for m in item['Mounts'] if m['Type'] == 'volume']))
|
||||
return sorted(result, key=lambda x: (x['service'], x['number']))
|
||||
|
||||
|
||||
def save(name, data):
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
(EVIDENCE / name).write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def wait_api(count=2, timeout=45):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
items = containers('api')
|
||||
if len(items) == count and all(x['health'] == 'healthy' for x in items):
|
||||
# DNS + one successful active health check; LB recovery is verified by HTTP tests.
|
||||
time.sleep(3)
|
||||
return items
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('API containers did not become healthy')
|
||||
|
||||
|
||||
def stop_one():
|
||||
target = os.environ.get('TARGET_INSTANCE_ID')
|
||||
items = containers('api')
|
||||
if len(items) < 2:
|
||||
raise RuntimeError('Need at least two live API replicas')
|
||||
selected = [x for x in items if target and target.startswith(x['hostname'] + '-')]
|
||||
if len(selected) != 1:
|
||||
raise RuntimeError('TARGET_INSTANCE_ID must identify one live API process')
|
||||
started = time.monotonic()
|
||||
command('docker', 'stop', '--time', '8', selected[0]['id'])
|
||||
after = containers('api', all_states=True)
|
||||
assert next(x for x in after if x['id'] == selected[0]['id'])['status'] == 'exited'
|
||||
assert len([x for x in after if x['status'] == 'running']) >= 1
|
||||
save('stop-one.json', dict(target_instance=target, stopped=selected[0], after=after,
|
||||
hook_seconds=time.monotonic() - started))
|
||||
print('Stopped selected API process; other API remains running. evidence/stop-one.json')
|
||||
|
||||
|
||||
def restart():
|
||||
before = containers()
|
||||
compose('restart', 'api', 'db', 'redis', 'lb')
|
||||
after = containers()
|
||||
for old in before:
|
||||
if old['service'] == 'migrate':
|
||||
continue
|
||||
new = next(x for x in after if x['id'] == old['id'])
|
||||
assert old['started_at'] != new['started_at'], old['name'] + ' did not restart'
|
||||
assert old['mounts'] == new['mounts'], 'Volumes changed'
|
||||
save('restart.json', dict(before=before, after=after))
|
||||
print('Restarted both API processes, PostgreSQL, Redis and LB; volumes unchanged.')
|
||||
|
||||
|
||||
def replace_one(identifier):
|
||||
# Pin exact Compose-owned ID before stopping/removing. No storage containers.
|
||||
old = next(x for x in containers('api') if x['id'] == identifier)
|
||||
command('docker', 'stop', '--time', '8', old['id'])
|
||||
command('docker', 'rm', old['id'])
|
||||
# Keep the other running replica, even if a newly built image is available.
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
||||
items = wait_api()
|
||||
# Compose may assign a new replica number; verify a new process exists.
|
||||
assert all(x['id'] != identifier for x in items)
|
||||
return dict(old=old, after=items)
|
||||
|
||||
|
||||
def rolling():
|
||||
initial = containers('api')
|
||||
assert len(initial) == 2, 'This grade 4 demonstration uses exactly two replicas'
|
||||
changes = []
|
||||
for old in initial:
|
||||
changes.append(replace_one(old['id']))
|
||||
save('rolling-processes.json', dict(before=initial, replacements=changes, after=containers('api')))
|
||||
print('Replaced each API container in turn; both have new container IDs.')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action', choices=['restart', 'stop-one', 'rolling', 'restore', 'state'])
|
||||
action = parser.parse_args().action
|
||||
if action == 'restart':
|
||||
restart()
|
||||
elif action == 'stop-one':
|
||||
stop_one()
|
||||
elif action == 'rolling':
|
||||
rolling()
|
||||
elif action == 'restore':
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
||||
wait_api()
|
||||
else:
|
||||
print(json.dumps(containers(all_states=True), indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
python -X utf8 scripts\labctl.py restart
|
||||
exit /b %errorlevel%
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/labctl.py restart
|
||||
@@ -0,0 +1,4 @@
|
||||
@echo off
|
||||
cd /d "%~dp0\.."
|
||||
python -X utf8 scripts\labctl.py stop-one
|
||||
exit /b %errorlevel%
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/labctl.py stop-one
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run public checks and all grade 4 demonstrations, preserving sanitized evidence."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / 'evidence'
|
||||
ENV = dict(os.environ, PYTHONUTF8='1', PYTHONIOENCODING='utf-8')
|
||||
BASE = os.environ.get('BASE_URL', 'http://127.0.0.1:8080')
|
||||
ENV['BASE_URL'] = BASE
|
||||
PYTHON = sys.executable
|
||||
|
||||
|
||||
def run(name, args):
|
||||
print('Running', name, flush=True)
|
||||
proc = subprocess.run(args, cwd=ROOT, env=ENV, text=True, encoding='utf-8', errors='replace',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
output = '$ ' + ' '.join(args).replace(PYTHON, 'python') + '\n' + proc.stdout
|
||||
output = '\n'.join(line.rstrip() for line in output.splitlines()).rstrip() + '\n'
|
||||
(EVIDENCE / (name + '.txt')).write_text(output, encoding='utf-8')
|
||||
print(proc.stdout, end='', flush=True)
|
||||
if proc.returncode:
|
||||
raise SystemExit('Check failed: ' + name)
|
||||
|
||||
|
||||
def main():
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
run('check', [PYTHON, 'scripts/check_contract.py'])
|
||||
run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet'])
|
||||
run('smoke', [PYTHON, 'tests/smoke.py', '--base-url', BASE])
|
||||
run('idempotency', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_idempotency.py'])
|
||||
run('migration-repeat', ['docker', 'compose', 'run', '--rm', '--no-deps', 'migrate'])
|
||||
run('persistence', [PYTHON, 'tests/resilience.py', '--mode', 'persistence', '--base-url', BASE,
|
||||
'--action-script', 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'])
|
||||
run('restore-before-failover', [PYTHON, 'scripts/labctl.py', 'restore'])
|
||||
run('failover', [PYTHON, 'tests/resilience.py', '--mode', 'failover', '--base-url', BASE,
|
||||
'--action-script', 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'])
|
||||
run('restore-after-failover', [PYTHON, 'scripts/labctl.py', 'restore'])
|
||||
run('load-grade4', [PYTHON, 'student-tests/test_recovery.py'])
|
||||
run('smoke-after', [PYTHON, 'tests/smoke.py', '--base-url', BASE])
|
||||
run('processes', [PYTHON, 'scripts/labctl.py', 'state'])
|
||||
info = json.loads(subprocess.check_output(['docker', 'info', '--format', '{{json .}}'], text=True))
|
||||
environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), host_os=platform.platform(),
|
||||
client_python=platform.python_version(),
|
||||
docker={k: info[k] for k in ('ServerVersion', 'OperatingSystem', 'KernelVersion',
|
||||
'NCPU', 'MemTotal', 'Architecture')}, images={})
|
||||
for name in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm', 'haproxy:3.0.16-alpine'):
|
||||
environment['images'][name] = json.loads(subprocess.check_output(
|
||||
['docker', 'image', 'inspect', name, '--format', '{{json .RepoDigests}}'], text=True))
|
||||
(EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: public checks and every grade 4 scenario complete.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1 @@
|
||||
"""Lab 1: bounded, single-process in-memory messenger."""
|
||||
@@ -0,0 +1,332 @@
|
||||
"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT."""
|
||||
import base64
|
||||
from dataclasses import dataclass, fields
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg.types.json import Jsonb
|
||||
from psycopg_pool import ConnectionPool
|
||||
import redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status, message):
|
||||
self.status, self.message = status, message
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
app_host: str = '0.0.0.0'
|
||||
app_port: int = 8080
|
||||
workers: int = 16
|
||||
request_queue: int = 32
|
||||
socket_timeout: int = 2
|
||||
max_body_bytes: int = 65536
|
||||
session_ttl_seconds: int = 3600
|
||||
idempotency_ttl_seconds: int = 86400
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
values = {}
|
||||
for field in fields(cls):
|
||||
raw = os.environ.get(field.name.upper(), field.default)
|
||||
values[field.name] = str(raw) if field.type is str else int(raw)
|
||||
if field.type is int and values[field.name] < 1:
|
||||
raise ValueError(field.name + ' must be positive')
|
||||
result = cls(**values)
|
||||
if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128:
|
||||
raise ValueError('port/workers/queue outside bounds')
|
||||
if result.max_body_bytes > 1048576 or result.socket_timeout > 5:
|
||||
raise ValueError('body <= 1 MiB and socket timeout <= 5 required')
|
||||
if os.getenv('SESSION_STORE', 'redis') != 'redis':
|
||||
raise ValueError('This solution uses Redis sessions')
|
||||
if result.idempotency_ttl_seconds < 86400:
|
||||
raise ValueError('Idempotency retention must be at least 24 hours')
|
||||
return result
|
||||
|
||||
|
||||
def secret_file(name):
|
||||
return Path(os.environ[name]).read_text(encoding='utf-8').strip()
|
||||
|
||||
|
||||
def db_connect_kwargs():
|
||||
return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 '
|
||||
'-c idle_in_transaction_session_timeout=5000', row_factory=dict_row)
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
if not isinstance(value, str) or not re.fullmatch(
|
||||
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value):
|
||||
raise APIError(400, 'Invalid UUID')
|
||||
return str(uuid.UUID(value))
|
||||
|
||||
|
||||
def string(value, maximum):
|
||||
if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value:
|
||||
raise APIError(400, 'Invalid string')
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise APIError(400, 'Invalid Unicode') from None
|
||||
return value
|
||||
|
||||
|
||||
def object_fields(body, expected):
|
||||
if not isinstance(body, dict) or set(body) != set(expected):
|
||||
raise APIError(400, 'Invalid object fields')
|
||||
|
||||
|
||||
def password_value(value):
|
||||
if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128:
|
||||
raise APIError(400, 'Invalid password length')
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise APIError(400, 'Invalid Unicode') from None
|
||||
return value
|
||||
|
||||
|
||||
def password_hash(value, salt=None):
|
||||
salt = salt or secrets.token_bytes(16)
|
||||
digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32)
|
||||
return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex()
|
||||
|
||||
|
||||
def password_matches(value, encoded):
|
||||
# Only this versioned format is generated by registration.
|
||||
salt = bytes.fromhex(encoded.split('$')[4])
|
||||
return hmac.compare_digest(password_hash(value, salt), encoded)
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def public_row(row):
|
||||
return {key: (str(value) if isinstance(value, uuid.UUID) else stamp(value)
|
||||
if isinstance(value, datetime) else value)
|
||||
for key, value in row.items() if key != 'password_hash'}
|
||||
|
||||
|
||||
USER_COLUMNS = 'id, username, display_name, created_at'
|
||||
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at'
|
||||
# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters.
|
||||
HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s '
|
||||
'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s')
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, config):
|
||||
self.config = config
|
||||
self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE'))
|
||||
if len(self.secret) < 32:
|
||||
raise ValueError('Cursor key must be at least 256 bits')
|
||||
self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8,
|
||||
timeout=1, max_waiting=16, kwargs=db_connect_kwargs(),
|
||||
check=ConnectionPool.check_connection, open=True)
|
||||
self.cache_pool = redis.ConnectionPool.from_url(
|
||||
secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5,
|
||||
socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0))
|
||||
self.cache = redis.Redis(connection_pool=self.cache_pool)
|
||||
self.dummy_hash = password_hash('dummy-password-for-timing')
|
||||
|
||||
def close(self):
|
||||
self.pool.close(timeout=2)
|
||||
self.cache_pool.disconnect()
|
||||
|
||||
def ready(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
self.cache.ping()
|
||||
|
||||
def create_user(self, body):
|
||||
object_fields(body, ('username', 'display_name', 'password'))
|
||||
username = string(body['username'], 32)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
|
||||
raise APIError(400, 'Invalid username')
|
||||
display = string(body['display_name'], 100)
|
||||
hashed = password_hash(password_value(body['password']))
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) '
|
||||
'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS,
|
||||
(uuid.uuid4(), username, display, hashed)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise APIError(409, 'Username already exists') from None
|
||||
return public_row(row)
|
||||
|
||||
def login(self, authorization):
|
||||
try:
|
||||
scheme, encoded = authorization.split(' ', 1)
|
||||
if scheme.lower() != 'basic' or len(encoded) > 1024:
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username):
|
||||
raise ValueError()
|
||||
password_value(password)
|
||||
except (AttributeError, ValueError, UnicodeError, APIError):
|
||||
raise APIError(401, 'Invalid credentials') from None
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone()
|
||||
matched = password_matches(password, row['password_hash'] if row else self.dummy_hash)
|
||||
if not matched or row is None:
|
||||
raise APIError(401, 'Invalid credentials')
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000
|
||||
self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms)
|
||||
return dict(token=token, token_type='Bearer',
|
||||
expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)),
|
||||
user=public_row(row))
|
||||
|
||||
@staticmethod
|
||||
def session_key(token):
|
||||
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||
|
||||
def auth(self, header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError()
|
||||
except (ValueError, AttributeError):
|
||||
raise APIError(401, 'Invalid session') from None
|
||||
key = self.session_key(token)
|
||||
actor = self.cache.get(key)
|
||||
if actor is None:
|
||||
raise APIError(401, 'Invalid session')
|
||||
return actor, key
|
||||
|
||||
def me(self, actor):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(401, 'Invalid session')
|
||||
return public_row(row)
|
||||
|
||||
def chat_access(self, actor, identifier):
|
||||
identifier = uuid_value(identifier)
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s',
|
||||
(identifier, actor)).fetchone()
|
||||
if row is None:
|
||||
raise APIError(404, 'Object not found')
|
||||
return identifier
|
||||
|
||||
@staticmethod
|
||||
def chat_row(conn, row):
|
||||
members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position',
|
||||
(row['id'],)).fetchall()
|
||||
return dict(public_row(row), member_ids=[str(x['user_id']) for x in members])
|
||||
|
||||
def chat(self, chat_id):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
|
||||
return self.chat_row(conn, row)
|
||||
|
||||
def create_chat(self, actor, body):
|
||||
object_fields(body, ('title', 'member_ids'))
|
||||
title = string(body['title'], 100)
|
||||
members = body['member_ids']
|
||||
if not isinstance(members, list) or not 1 <= len(members) <= 100:
|
||||
raise APIError(400, 'Invalid members')
|
||||
members = [uuid_value(x) for x in members]
|
||||
if len(members) != len(set(members)) or actor not in members:
|
||||
raise APIError(400, 'Invalid members')
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *',
|
||||
(uuid.uuid4(), title)).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)',
|
||||
[(row['id'], member, pos) for pos, member in enumerate(members)])
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise APIError(400, 'Invalid members') from None
|
||||
return dict(public_row(row), member_ids=members)
|
||||
|
||||
def create_message(self, actor, chat_id, body, key):
|
||||
if not isinstance(key, str) or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
raise APIError(400, 'One valid Idempotency-Key is required')
|
||||
if not isinstance(body, dict) or set(body) not in ({'text'}, {'text', 'attachment_ids'}):
|
||||
raise APIError(400, 'Invalid object fields')
|
||||
# Attachments start at lab4. Empty and omitted arrays are equivalent here.
|
||||
if 'attachment_ids' in body and body['attachment_ids'] != []:
|
||||
raise APIError(400, 'Attachments are not supported in lab3')
|
||||
text = string(body['text'], 4000)
|
||||
digest = hashlib.sha256(json.dumps({'text': text, 'attachment_ids': []},
|
||||
ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
||||
scope = json.dumps([actor, 'POST', '/api/v1/chats/' + chat_id + '/messages', key])
|
||||
lock = int.from_bytes(hashlib.sha256(scope.encode()).digest()[:8], 'big', signed=True)
|
||||
with self.pool.connection() as conn:
|
||||
# Transaction lock spans SELECT, message INSERT, result INSERT and COMMIT.
|
||||
# All replicas use PostgreSQL; a timeout rolls back and may be retried.
|
||||
conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,))
|
||||
previous = conn.execute('SELECT fingerprint, response, expires_at > clock_timestamp() AS valid '
|
||||
'FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s',
|
||||
(actor, chat_id, key)).fetchone()
|
||||
if previous and previous['valid']:
|
||||
if previous['fingerprint'] != digest:
|
||||
raise APIError(409, 'Idempotency-Key already used with another body')
|
||||
return previous['response']
|
||||
if previous:
|
||||
conn.execute('DELETE FROM message_requests WHERE actor_id=%s AND chat_id=%s AND key=%s',
|
||||
(actor, chat_id, key))
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING ' + MESSAGE_COLUMNS, (uuid.uuid4(), chat_id, actor, text)).fetchone()
|
||||
result = public_row(row)
|
||||
conn.execute('INSERT INTO message_requests(actor_id,chat_id,key,fingerprint,response,expires_at) '
|
||||
"VALUES (%s,%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')",
|
||||
(actor, chat_id, key, digest, Jsonb(result), self.config.idempotency_ttl_seconds))
|
||||
return result
|
||||
|
||||
def encode_cursor(self, scope, key):
|
||||
payload = json.dumps([1, scope, *key], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode()
|
||||
|
||||
def decode_cursor(self, cursor, scope):
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor):
|
||||
raise ValueError()
|
||||
raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True)
|
||||
if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor:
|
||||
raise ValueError()
|
||||
payload, signature = raw[:-32], raw[-32:]
|
||||
if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')):
|
||||
raise ValueError()
|
||||
version, resource, timestamp, identifier = json.loads(payload)
|
||||
if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'):
|
||||
raise ValueError()
|
||||
return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier)
|
||||
except (ValueError, TypeError, UnicodeError, APIError):
|
||||
raise APIError(400, 'Invalid cursor') from None
|
||||
|
||||
def page(self, actor, chat_id, query):
|
||||
if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()):
|
||||
raise APIError(400, 'Invalid query')
|
||||
limit = query.get('limit', ['50'])[0]
|
||||
if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100:
|
||||
raise APIError(400, 'Invalid limit')
|
||||
limit = int(limit)
|
||||
scope = 'messages:' + chat_id if chat_id else 'chats:' + actor
|
||||
key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else (
|
||||
datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000')
|
||||
with self.pool.connection() as conn:
|
||||
if chat_id:
|
||||
rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall()
|
||||
items = [public_row(row) for row in rows[:limit]]
|
||||
else:
|
||||
rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) '
|
||||
'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall()
|
||||
items = [self.chat_row(conn, row) for row in rows[:limit]]
|
||||
cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None
|
||||
return dict(items=items, next_cursor=cursor)
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Transactional, versioned migrations; repeat application is a no-op."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import psycopg
|
||||
from .core import secret_file, db_connect_kwargs
|
||||
|
||||
|
||||
def main():
|
||||
with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(2026, 2)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations '
|
||||
'(name text PRIMARY KEY, sha256 text NOT NULL, '
|
||||
'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
|
||||
for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')):
|
||||
content = path.read_bytes()
|
||||
digest = hashlib.sha256(content).hexdigest()
|
||||
row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone()
|
||||
if row:
|
||||
if row['sha256'] != digest:
|
||||
raise RuntimeError('Applied migration checksum mismatch')
|
||||
print('Already applied:', path.name)
|
||||
continue
|
||||
conn.execute(content.decode('utf-8'), prepare=False)
|
||||
conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest))
|
||||
print('Applied:', path.name)
|
||||
print('Migrations committed.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies."""
|
||||
from collections import Counter
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
import uuid
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
import redis
|
||||
|
||||
from .core import APIError, Config, Store
|
||||
|
||||
|
||||
CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict",
|
||||
413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited",
|
||||
503: "unavailable", 500: "internal_error"}
|
||||
|
||||
|
||||
def error_body(status, message, request_id):
|
||||
return {"error": {"code": CODES.get(status, "invalid_request"),
|
||||
"message": message, "request_id": request_id}}
|
||||
|
||||
|
||||
def log_event(**data):
|
||||
logging.info(json.dumps(data, ensure_ascii=True))
|
||||
|
||||
|
||||
class Server(HTTPServer):
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, config):
|
||||
self.config, self.store = config, Store(config)
|
||||
self.instance_id = os.environ.get('HOSTNAME', 'api') + '-' + uuid.uuid4().hex[:12]
|
||||
self.jobs = queue.Queue(maxsize=config.request_queue)
|
||||
self.draining = threading.Event()
|
||||
self.stats_lock = threading.Lock()
|
||||
self.statuses = Counter()
|
||||
self.active = 0
|
||||
self.clients = set()
|
||||
self.started = time.monotonic()
|
||||
self.request_queue_size = config.request_queue
|
||||
super().__init__((config.app_host, config.app_port), Handler)
|
||||
self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True)
|
||||
for n in range(config.workers)]
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def get_request(self):
|
||||
client, address = super().get_request()
|
||||
client.settimeout(self.config.socket_timeout)
|
||||
return client, address
|
||||
|
||||
def record(self, status):
|
||||
with self.stats_lock:
|
||||
self.statuses[str(status)] += 1
|
||||
|
||||
def process_request(self, request, address):
|
||||
try:
|
||||
if self.draining.is_set():
|
||||
raise queue.Full()
|
||||
self.jobs.put_nowait((request, address))
|
||||
except queue.Full:
|
||||
# Transport admission before HTTP parsing: bounded time, no extra thread.
|
||||
identifier = str(uuid.uuid4())
|
||||
body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode()
|
||||
headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n"
|
||||
"Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier +
|
||||
"\r\nX-Instance-Id: " + self.instance_id +
|
||||
"\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode()
|
||||
try:
|
||||
request.settimeout(.05)
|
||||
# Wait for the first request bytes: closing before a local client
|
||||
# finishes sendall can otherwise discard the 503 with a TCP reset.
|
||||
try:
|
||||
request.recv(16384)
|
||||
except OSError:
|
||||
pass
|
||||
request.sendall(headers + body)
|
||||
request.shutdown(socket.SHUT_WR)
|
||||
request.settimeout(.01)
|
||||
for _ in range(5):
|
||||
if not request.recv(16384):
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
self.shutdown_request(request)
|
||||
self.record(503)
|
||||
log_event(event="queue_rejected", status=503, request_id=identifier)
|
||||
|
||||
def worker(self):
|
||||
while not self.draining.is_set() or not self.jobs.empty():
|
||||
try:
|
||||
request, address = self.jobs.get(timeout=.1)
|
||||
except queue.Empty:
|
||||
continue
|
||||
with self.stats_lock:
|
||||
self.active += 1
|
||||
self.clients.add(request)
|
||||
try:
|
||||
self.finish_request(request, address)
|
||||
except OSError:
|
||||
pass
|
||||
except Exception:
|
||||
log_event(event="worker_error")
|
||||
finally:
|
||||
self.shutdown_request(request)
|
||||
with self.stats_lock:
|
||||
self.active -= 1
|
||||
self.clients.discard(request)
|
||||
self.jobs.task_done()
|
||||
|
||||
def server_close(self):
|
||||
self.draining.set()
|
||||
super().server_close()
|
||||
# Stop listening, reject queued work, let accepted in-flight work COMMIT
|
||||
# and send its response before forcibly closing any remaining socket.
|
||||
deadline = time.monotonic() + 6
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=max(0, deadline - time.monotonic()))
|
||||
with self.stats_lock:
|
||||
clients = list(self.clients)
|
||||
for client in clients:
|
||||
try:
|
||||
client.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
while True:
|
||||
try:
|
||||
request, _ = self.jobs.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
self.shutdown_request(request)
|
||||
self.jobs.task_done()
|
||||
self.store.close()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "Messenger"
|
||||
sys_version = ""
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
self.request_id = str(uuid.uuid4())
|
||||
self.received = time.monotonic()
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def parse_request(self):
|
||||
original = self.rfile
|
||||
|
||||
class HeaderBudget:
|
||||
remaining = 16384
|
||||
|
||||
def readline(self, limit=-1):
|
||||
line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1)
|
||||
self.remaining -= len(line)
|
||||
if self.remaining < 0:
|
||||
raise http.client.LineTooLong("header section")
|
||||
return line
|
||||
|
||||
self.rfile = HeaderBudget()
|
||||
try:
|
||||
return super().parse_request()
|
||||
finally:
|
||||
self.rfile = original
|
||||
|
||||
def send_error(self, code, message=None, explain=None):
|
||||
self.reply(code if code in CODES else 400, error_body(
|
||||
code if code in CODES else 400, "Invalid HTTP request", self.request_id))
|
||||
|
||||
def reply(self, status, body):
|
||||
raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8')
|
||||
content_type = 'application/json'
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(raw)))
|
||||
self.send_header("X-Request-Id", self.request_id)
|
||||
self.send_header("X-Instance-Id", self.server.instance_id)
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Connection", "close")
|
||||
if status == 401:
|
||||
challenge = ('Basic realm="messenger", charset="UTF-8"'
|
||||
if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
|
||||
self.send_header('WWW-Authenticate', challenge)
|
||||
if status in (429, 503):
|
||||
self.send_header("Retry-After", "1")
|
||||
self.end_headers()
|
||||
self.close_connection = True
|
||||
self.wfile.write(raw)
|
||||
self.server.record(status)
|
||||
# No URL query, header, username, text, or exception detail in logs.
|
||||
log_event(event="request", method=self.command, status=status,
|
||||
request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3))
|
||||
|
||||
def handle_expect_100(self):
|
||||
# Do not acknowledge a large unauthorized request before checking identity.
|
||||
return True
|
||||
|
||||
def body(self):
|
||||
if self.headers.get_all("Transfer-Encoding"):
|
||||
raise APIError(400, "Transfer-Encoding is unsupported")
|
||||
sizes = self.headers.get_all("Content-Length", [])
|
||||
if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10:
|
||||
raise APIError(400, "One valid Content-Length is required")
|
||||
size = int(sizes[0])
|
||||
if size > self.server.config.max_body_bytes:
|
||||
raise APIError(413, "Request body exceeds byte limit")
|
||||
if self.headers.get_content_type() != "application/json":
|
||||
raise APIError(415, "Expected application/json")
|
||||
if self.headers.get("Expect", "").lower() == "100-continue":
|
||||
self.send_response_only(100)
|
||||
self.end_headers()
|
||||
raw = self.rfile.read(size)
|
||||
if len(raw) != size:
|
||||
raise APIError(400, "Incomplete request body")
|
||||
|
||||
def pairs(values):
|
||||
result = {}
|
||||
for key, value in values:
|
||||
if key in result:
|
||||
raise ValueError("duplicate key")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
def constant(_):
|
||||
raise ValueError("non JSON constant")
|
||||
|
||||
try:
|
||||
return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant)
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
raise APIError(400, "Invalid JSON") from None
|
||||
|
||||
def dispatch(self):
|
||||
try:
|
||||
url = urlsplit(self.path)
|
||||
except ValueError:
|
||||
raise APIError(400, "Invalid request target") from None
|
||||
path, method = url.path, self.command
|
||||
store = self.server.store
|
||||
if self.server.draining.is_set():
|
||||
raise APIError(503, "Shutting down")
|
||||
if method == 'GET' and path == '/health/live':
|
||||
return 200, {'status': 'ok'}
|
||||
if method == 'GET' and path == '/health/ready':
|
||||
store.ready()
|
||||
return 200, {'status': 'ok'}
|
||||
if path == '/api/v1/users' and method == 'POST':
|
||||
return 201, store.create_user(self.body())
|
||||
headers = self.headers.get_all('Authorization', [])
|
||||
authorization = headers[0] if len(headers) == 1 else None
|
||||
if path == '/api/v1/auth/sessions' and method == 'POST':
|
||||
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0':
|
||||
raise APIError(400, 'Login takes no request body')
|
||||
return 201, store.login(authorization)
|
||||
actor, session_key = store.auth(authorization)
|
||||
if method == 'DELETE' and path == '/api/v1/auth/sessions/current':
|
||||
store.cache.delete(session_key)
|
||||
return 204, None
|
||||
if method == 'GET' and path == '/api/v1/users/me':
|
||||
return 200, store.me(actor)
|
||||
parts = path.strip("/").split("/")
|
||||
chat_id = None
|
||||
if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]:
|
||||
chat_id = store.chat_access(actor, parts[3])
|
||||
if method == "POST" and path == "/api/v1/chats":
|
||||
return 201, store.create_chat(actor, self.body())
|
||||
if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages":
|
||||
keys = self.headers.get_all('Idempotency-Key', [])
|
||||
key = keys[0] if len(keys) == 1 else None
|
||||
return 201, store.create_message(actor, chat_id, self.body(), key)
|
||||
if method == "GET":
|
||||
if chat_id and len(parts) == 4:
|
||||
return 200, store.chat(chat_id)
|
||||
if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"):
|
||||
try:
|
||||
query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True, max_num_fields=8)
|
||||
except ValueError:
|
||||
raise APIError(400, "Invalid query") from None
|
||||
return 200, store.page(actor, chat_id, query)
|
||||
raise APIError(404, "Object not found")
|
||||
|
||||
def handle_api(self):
|
||||
try:
|
||||
status, result = self.dispatch()
|
||||
except APIError as error:
|
||||
status = error.status
|
||||
result = error_body(status, error.message, self.request_id)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled,
|
||||
psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError):
|
||||
status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id)
|
||||
except (TimeoutError, socket.timeout):
|
||||
status, result = 400, error_body(400, "Request read timeout", self.request_id)
|
||||
except Exception:
|
||||
status, result = 500, error_body(500, "Internal error", self.request_id)
|
||||
log_event(event="internal_error", request_id=self.request_id)
|
||||
try:
|
||||
self.reply(status, result)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
# Pool diagnostics may include connection details; application emits sanitized errors.
|
||||
logging.getLogger('psycopg.pool').disabled = True
|
||||
config = Config.from_env()
|
||||
server = Server(config)
|
||||
|
||||
def stop(signum, _):
|
||||
if not server.draining.is_set():
|
||||
server.draining.set()
|
||||
log_event(event="shutdown_started", signal=signum)
|
||||
threading.Thread(target=server.shutdown, daemon=True).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, stop)
|
||||
signal.signal(signal.SIGINT, stop)
|
||||
log_event(event="started", instance_id=server.instance_id, port=config.app_port, workers=config.workers)
|
||||
try:
|
||||
server.serve_forever(poll_interval=.1)
|
||||
finally:
|
||||
server.server_close()
|
||||
log_event(event="shutdown_complete")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Run inside API network: same session and simultaneous requests at distinct processes."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def main():
|
||||
clients = [Client('http://' + ip + ':8080') for ip in sorted(set(socket.gethostbyname_ex('api')[2]))]
|
||||
check(len(clients) == 2, 'Expected two API addresses in Docker DNS')
|
||||
c = clients[0]
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(16)
|
||||
|
||||
def send(n):
|
||||
barrier.wait(timeout=10)
|
||||
return clients[n % 2].call('POST', path, 201, actor=alice,
|
||||
body={'attachment_ids': [], 'text': 'Параллельный повтор'} if n % 2 else
|
||||
{'text': 'Параллельный повтор'},
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
responses = list(pool.map(send, range(16)))
|
||||
messages = [x['json'] for x in responses]
|
||||
instances = sorted({x['headers']['X-Instance-Id'] for x in responses})
|
||||
check(len(instances) == 2, 'Requests did not reach distinct API processes')
|
||||
check(all(x == messages[0] for x in messages), 'Concurrent responses differ')
|
||||
history = clients[1].call('GET', path, actor=bob, schema='MessagePage')['json']['items']
|
||||
check(history == [messages[0]], 'Concurrent requests created duplicates')
|
||||
clients[1].call('POST', path, 409, actor=alice, body={'text': 'Другой body'}, headers={'Idempotency-Key': key})
|
||||
# Scope is user + chat; an independent user and another chat may reuse the key.
|
||||
other = clients[1].message(bob, chat, 'Другой пользователь', key=key)['json']
|
||||
chat2 = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
third = clients[1].message(alice, chat2, 'Другой чат', key=key)['json']
|
||||
check(len({messages[0]['id'], other['id'], third['id']}) == 3, 'Incorrect idempotency scope')
|
||||
for invalid in ('', 'contains space', 'x' * 129):
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'Invalid key'}, headers={'Idempotency-Key': invalid})
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
clients[1].call('POST', path, 401, actor=alice, body={'text': 'Параллельный повтор'},
|
||||
headers={'Idempotency-Key': key})
|
||||
print(json.dumps(dict(result='PASS', parallel_requests=16, api_instances=instances,
|
||||
unique_messages=1, scope_conflict_logout='PASS'), ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,173 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 4 only: each process failure and sequential replacement under one fixed load."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path[:0] = [str(ROOT / 'tests'), str(ROOT / 'scripts')]
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
from labctl import command, compose, containers, replace_one, save, wait_api
|
||||
|
||||
# IPv4-only published LB: avoid slow localhost IPv6 fallback on Windows.
|
||||
os.environ.setdefault('BASE_URL', 'http://127.0.0.1:8080')
|
||||
|
||||
|
||||
def quantile(values, q):
|
||||
ordered = sorted(values)
|
||||
return round(ordered[min(len(ordered) - 1, int((len(ordered) - 1) * q))] * 1000, 3) if values else None
|
||||
|
||||
|
||||
def scenario(mode, target=None, seconds=32):
|
||||
c = Client(timeout=2)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
original_key = uuid.uuid4().hex
|
||||
original = c.message(alice, chat, 'Подтверждено до воздействия', key=original_key)['json']
|
||||
instances_before = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
for _ in range(20)}
|
||||
check(len(instances_before) == 2, 'Need two distinct processes before experiment')
|
||||
samples, confirmations, resources, action = [], [], [], {}
|
||||
stop = threading.Event()
|
||||
start = time.monotonic()
|
||||
|
||||
def worker(n):
|
||||
client = Client(timeout=2)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load-' + key
|
||||
while not stop.is_set():
|
||||
for operation in ('write', 'read'):
|
||||
at = time.monotonic()
|
||||
try:
|
||||
if operation == 'write':
|
||||
result = client.message(alice, chat, text, key=key)
|
||||
confirmations.append(result['json'])
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load-' + key
|
||||
else:
|
||||
result = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
||||
actor=bob, schema='MessagePage')
|
||||
check(original in result['json']['items'], 'Confirmed data missing during action')
|
||||
samples.append(dict(at=round(at - start, 5), end=round(time.monotonic() - start, 5),
|
||||
elapsed=time.monotonic() - at, operation=operation,
|
||||
ok=True, status=result['status'], instance=result['headers']['X-Instance-Id']))
|
||||
except Failure as exc:
|
||||
# Failure only contains sanitized method/path/status or transport type.
|
||||
samples.append(dict(at=round(at - start, 5), end=round(time.monotonic() - start, 5),
|
||||
elapsed=time.monotonic() - at, operation=operation,
|
||||
ok=False, error=str(exc)))
|
||||
stop.wait(.03)
|
||||
|
||||
def affect():
|
||||
stop.wait(5)
|
||||
action['start'] = time.monotonic() - start
|
||||
try:
|
||||
if mode == 'stop':
|
||||
before = containers('api')
|
||||
command('docker', 'stop', '--time', '8', target['id'])
|
||||
action['stop_completed'] = time.monotonic() - start
|
||||
after = containers('api', all_states=True)
|
||||
check(next(x for x in after if x['id'] == target['id'])['status'] == 'exited', 'Target still running')
|
||||
action.update(before=before, after_stop=after)
|
||||
stop.wait(10)
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', 'api')
|
||||
wait_api()
|
||||
else:
|
||||
initial = containers('api')
|
||||
changes = []
|
||||
for item in initial:
|
||||
changes.append(replace_one(item['id']))
|
||||
action['replacements'] = changes
|
||||
save('rolling-processes.json', dict(before=initial, replacements=changes, after=containers('api')))
|
||||
action['end'] = time.monotonic() - start
|
||||
except Exception as exc:
|
||||
action['error'] = type(exc).__name__
|
||||
stop.set()
|
||||
|
||||
def monitor():
|
||||
while not stop.is_set():
|
||||
ids = [x['id'] for x in containers()]
|
||||
rows = command('docker', 'stats', '--no-stream', '--format', '{{json .}}', *ids)
|
||||
resources.append(dict(at=round(time.monotonic() - start, 3),
|
||||
containers=[json.loads(x) for x in rows.splitlines()]))
|
||||
stop.wait(5)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=6) as pool:
|
||||
workers = [pool.submit(worker, n) for n in range(4)]
|
||||
impact = pool.submit(affect)
|
||||
monitor_job = pool.submit(monitor)
|
||||
stop.wait(seconds)
|
||||
stop.set()
|
||||
impact.result()
|
||||
monitor_job.result()
|
||||
for job in workers:
|
||||
job.result()
|
||||
check('error' not in action, 'Action failed: ' + action.get('error', ''))
|
||||
wait_api()
|
||||
all_messages = history(c, bob, chat)
|
||||
by_id = {x['id']: x for x in all_messages}
|
||||
check(len(by_id) == len(all_messages), 'Duplicate IDs')
|
||||
check(len({x['text'] for x in all_messages}) == len(all_messages), 'Uncertain retry duplicated a message')
|
||||
check(all(by_id.get(x['id']) == x for x in [original, *confirmations]), 'Confirmed data lost')
|
||||
check(c.message(alice, chat, original['text'], key=original_key)['json'] == original,
|
||||
'Pre-action idempotency result changed after replacement')
|
||||
elapsed = max(x['end'] for x in samples)
|
||||
errors = [x for x in samples if not x['ok']]
|
||||
successful = [x for x in samples if x['ok']]
|
||||
latencies = [x['elapsed'] for x in successful]
|
||||
result = dict(mode=mode, target=target['name'] if target else None,
|
||||
at_utc=datetime.now(timezone.utc).isoformat(), duration_seconds=round(elapsed, 3),
|
||||
warmup_seconds=5, concurrency=4, profile='each worker: POST + GET, then 30ms pause; closed loop',
|
||||
initial_messages=1, final_messages=len(all_messages), confirmations=len(confirmations),
|
||||
operations=len(samples), successful_operations=len(successful), errors=len(errors),
|
||||
error_percent=round(100 * len(errors) / len(samples), 3),
|
||||
successful_rps=round(len(successful) / elapsed, 3),
|
||||
latency_success_ms=dict(p50=quantile(latencies, .5), p95=quantile(latencies, .95), p99=quantile(latencies, .99)),
|
||||
instances_before=sorted(instances_before), action=action, resources=resources,
|
||||
confirmed_data_lost=0, duplicate_messages=0)
|
||||
if mode == 'stop':
|
||||
done = action['stop_completed']
|
||||
post = [x for x in samples if done <= x['at'] < done + 8]
|
||||
recovery = {}
|
||||
for op in ('read', 'write'):
|
||||
valid = [x for x in post if x['ok'] and x['operation'] == op]
|
||||
check(bool(valid), op + ' did not continue after stop')
|
||||
recovery[op] = round(min(x['end'] for x in valid) - action['start'], 3)
|
||||
check(recovery[op] <= 10, 'Recovery exceeded 10 seconds')
|
||||
relevant_errors = [x for x in errors if action['start'] <= x['at'] < done + 8]
|
||||
result['first_success_after_stop_seconds_from_action_start'] = recovery
|
||||
result['last_error_seconds_from_action_start'] = round(
|
||||
max((x['end'] - action['start'] for x in relevant_errors), default=0), 3)
|
||||
result['successful_rps_8s_after_stop'] = round(sum(x['ok'] for x in post) / 8, 3)
|
||||
else:
|
||||
check(len(action['replacements']) == 2, 'Both containers must be replaced')
|
||||
check(len(errors) / len(samples) < .2, 'Too many errors during sequential replacement')
|
||||
filename = 'load-' + (target['name'] if target else 'rolling') + '.json'
|
||||
result['samples'] = samples
|
||||
save(filename, result)
|
||||
print(json.dumps({k: v for k, v in result.items() if k not in ('samples', 'resources', 'action')}, ensure_ascii=False))
|
||||
print('Evidence:', 'evidence/' + filename, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--mode', choices=['failures', 'rolling', 'all'], default='all')
|
||||
args = p.parse_args()
|
||||
wait_api()
|
||||
if args.mode in ('failures', 'all'):
|
||||
for target in containers('api'):
|
||||
scenario('stop', target)
|
||||
if args.mode in ('rolling', 'all'):
|
||||
scenario('rolling')
|
||||
print('PASS: grade 4 load scenarios; no scale or multi-node research track.')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user