From 7088791d5c9ee07048f7100a84dd83e770a8eaa1 Mon Sep 17 00:00:00 2001 From: sashadugina06 <102163632+sashadugina06@users.noreply.github.com> Date: Fri, 6 Nov 2026 01:43:33 +0300 Subject: [PATCH 1/2] feat: implement lab3 API load balancing and failover --- .dockerignore | 2 + .env.example | 14 +- .gitignore | 2 + Dockerfile | 30 + REPORT.md | 245 +++++++- compose.yaml | 91 +++ db/migrations/001_initial.sql | 51 ++ db/migrations/002_idempotency.sql | 9 + evidence/README.md | 44 +- evidence/auth.txt | 26 + evidence/changed-files.txt | 80 +++ evidence/check.txt | 2 + evidence/compose-validation.txt | 1 + evidence/containers.json | 118 ++++ evidence/containers.txt | 7 + evidence/dataset.txt | 4 + evidence/defense-ignore.txt | 1 + evidence/demo.txt | 47 ++ evidence/diff-stat.txt | 6 + evidence/dns-change.json | 14 + evidence/environment.txt | 19 + evidence/failover-public.txt | 4 + evidence/failover-replica-1.json | 305 ++++++++++ evidence/failover-replica-1.txt | 80 +++ evidence/failover-replica-2.json | 305 ++++++++++ evidence/failover-replica-2.txt | 36 ++ evidence/fixes-20261106/README.md | 45 ++ evidence/fixes-20261106/async-audit.md | 40 ++ evidence/fixes-20261106/build.txt | 58 ++ evidence/fixes-20261106/check.txt | 2 + evidence/fixes-20261106/containers-final.json | 118 ++++ evidence/fixes-20261106/containers-final.txt | 7 + evidence/fixes-20261106/deploy.txt | 2 + evidence/fixes-20261106/event-loop-after.txt | 12 + evidence/fixes-20261106/event-loop-before.txt | 153 +++++ evidence/fixes-20261106/failover-public.txt | 4 + .../fixes-20261106/failover-replica-1.json | 296 ++++++++++ .../fixes-20261106/failover-replica-1.txt | 38 ++ .../fixes-20261106/failover-replica-2.json | 296 ++++++++++ .../fixes-20261106/failover-replica-2.txt | 38 ++ evidence/fixes-20261106/faults-after.json | 118 ++++ evidence/fixes-20261106/faults-before.json | 118 ++++ evidence/fixes-20261106/git-status.txt | 68 +++ evidence/fixes-20261106/idempotency.json | 24 + evidence/fixes-20261106/idempotency.txt | 24 + evidence/fixes-20261106/implementation.patch | 50 ++ .../infrastructure-isolation.txt | 3 + evidence/fixes-20261106/integrity.txt | 6 + evidence/fixes-20261106/load-after.json | 17 + evidence/fixes-20261106/load-after.txt | 17 + evidence/fixes-20261106/load-before.json | 17 + evidence/fixes-20261106/load-before.txt | 17 + evidence/fixes-20261106/metadata.json | 33 ++ .../fixes-20261106/persistence-after.json | 118 ++++ .../fixes-20261106/persistence-before.json | 118 ++++ evidence/fixes-20261106/persistence.txt | 3 + evidence/fixes-20261106/rolling-update.json | 298 ++++++++++ evidence/fixes-20261106/rolling-update.txt | 58 ++ evidence/fixes-20261106/smoke.txt | 2 + evidence/fixes-20261106/swagger.txt | 2 + evidence/fixes-20261106/validate.txt | 2 + evidence/git-status.txt | 67 +++ evidence/graceful-shutdown.txt | 22 + evidence/idempotency.json | 24 + evidence/lb-events.txt | 144 +++++ evidence/load-baseline-resources.json | 542 ++++++++++++++++++ evidence/load-baseline.json | 17 + evidence/load-baseline.txt | 17 + evidence/local-integrity.txt | 6 + evidence/migrations-after.txt | 5 + evidence/migrations-before.txt | 5 + evidence/migrations-initial.txt | 4 + evidence/migrations-rerun.txt | 2 + evidence/persistence-containers-after.json | 118 ++++ evidence/persistence-containers-before.json | 118 ++++ evidence/persistence.txt | 3 + evidence/readiness.json | 26 + evidence/resources-summary.json | 107 ++++ evidence/rolling-continuity.json | 407 +++++++++++++ evidence/rolling-continuity.txt | 58 ++ evidence/rolling-preparation.txt | 100 ++++ evidence/rolling-update.json | 296 ++++++++++ evidence/rolling-update.txt | 56 ++ evidence/smoke.txt | 2 + evidence/swagger.txt | 2 + evidence/validate.txt | 2 + infra/haproxy.cfg | 35 ++ requirements.txt | 7 + scripts/bootstrap.py | 31 + scripts/capture-state.py | 19 + scripts/compose.sh | 7 + scripts/demo.py | 53 ++ scripts/restart.sh | 5 + scripts/restore.sh | 7 + scripts/rolling-update.py | 79 +++ scripts/rolling-update.sh | 4 + scripts/stop-one.sh | 15 + scripts/summarize-resources.py | 31 + scripts/up.sh | 9 + scripts/verify.sh | 26 + scripts/wait-ready.py | 19 + src/__init__.py | 0 src/auth.py | 91 +++ src/chat_repository.py | 240 ++++++++ src/database.py | 48 ++ src/main.py | 496 ++++++++++++++++ src/migrate.py | 16 + src/session_cache.py | 48 ++ src/user_repository.py | 136 +++++ student-tests/README.md | 43 ++ student-tests/baseline.py | 21 + student-tests/check_auth.py | 150 +++++ student-tests/dns.py | 64 +++ student-tests/readiness.py | 46 ++ student-tests/scenarios.py | 216 +++++++ student-tests/test_event_loop.py | 132 +++++ 116 files changed, 7971 insertions(+), 38 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 db/migrations/001_initial.sql create mode 100644 db/migrations/002_idempotency.sql create mode 100644 evidence/auth.txt create mode 100644 evidence/changed-files.txt create mode 100644 evidence/check.txt create mode 100644 evidence/compose-validation.txt create mode 100644 evidence/containers.json create mode 100644 evidence/containers.txt create mode 100644 evidence/dataset.txt create mode 100644 evidence/defense-ignore.txt create mode 100644 evidence/demo.txt create mode 100644 evidence/diff-stat.txt create mode 100644 evidence/dns-change.json create mode 100644 evidence/environment.txt create mode 100644 evidence/failover-public.txt create mode 100644 evidence/failover-replica-1.json create mode 100644 evidence/failover-replica-1.txt create mode 100644 evidence/failover-replica-2.json create mode 100644 evidence/failover-replica-2.txt create mode 100644 evidence/fixes-20261106/README.md create mode 100644 evidence/fixes-20261106/async-audit.md create mode 100644 evidence/fixes-20261106/build.txt create mode 100644 evidence/fixes-20261106/check.txt create mode 100644 evidence/fixes-20261106/containers-final.json create mode 100644 evidence/fixes-20261106/containers-final.txt create mode 100644 evidence/fixes-20261106/deploy.txt create mode 100644 evidence/fixes-20261106/event-loop-after.txt create mode 100644 evidence/fixes-20261106/event-loop-before.txt create mode 100644 evidence/fixes-20261106/failover-public.txt create mode 100644 evidence/fixes-20261106/failover-replica-1.json create mode 100644 evidence/fixes-20261106/failover-replica-1.txt create mode 100644 evidence/fixes-20261106/failover-replica-2.json create mode 100644 evidence/fixes-20261106/failover-replica-2.txt create mode 100644 evidence/fixes-20261106/faults-after.json create mode 100644 evidence/fixes-20261106/faults-before.json create mode 100644 evidence/fixes-20261106/git-status.txt create mode 100644 evidence/fixes-20261106/idempotency.json create mode 100644 evidence/fixes-20261106/idempotency.txt create mode 100644 evidence/fixes-20261106/implementation.patch create mode 100644 evidence/fixes-20261106/infrastructure-isolation.txt create mode 100644 evidence/fixes-20261106/integrity.txt create mode 100644 evidence/fixes-20261106/load-after.json create mode 100644 evidence/fixes-20261106/load-after.txt create mode 100644 evidence/fixes-20261106/load-before.json create mode 100644 evidence/fixes-20261106/load-before.txt create mode 100644 evidence/fixes-20261106/metadata.json create mode 100644 evidence/fixes-20261106/persistence-after.json create mode 100644 evidence/fixes-20261106/persistence-before.json create mode 100644 evidence/fixes-20261106/persistence.txt create mode 100644 evidence/fixes-20261106/rolling-update.json create mode 100644 evidence/fixes-20261106/rolling-update.txt create mode 100644 evidence/fixes-20261106/smoke.txt create mode 100644 evidence/fixes-20261106/swagger.txt create mode 100644 evidence/fixes-20261106/validate.txt create mode 100644 evidence/git-status.txt create mode 100644 evidence/graceful-shutdown.txt create mode 100644 evidence/idempotency.json create mode 100644 evidence/lb-events.txt create mode 100644 evidence/load-baseline-resources.json create mode 100644 evidence/load-baseline.json create mode 100644 evidence/load-baseline.txt create mode 100644 evidence/local-integrity.txt create mode 100644 evidence/migrations-after.txt create mode 100644 evidence/migrations-before.txt create mode 100644 evidence/migrations-initial.txt create mode 100644 evidence/migrations-rerun.txt create mode 100644 evidence/persistence-containers-after.json create mode 100644 evidence/persistence-containers-before.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/readiness.json create mode 100644 evidence/resources-summary.json create mode 100644 evidence/rolling-continuity.json create mode 100644 evidence/rolling-continuity.txt create mode 100644 evidence/rolling-preparation.txt create mode 100644 evidence/rolling-update.json create mode 100644 evidence/rolling-update.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/swagger.txt create mode 100644 evidence/validate.txt create mode 100644 infra/haproxy.cfg create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100644 scripts/capture-state.py create mode 100755 scripts/compose.sh create mode 100755 scripts/demo.py create mode 100755 scripts/restart.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/rolling-update.py create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/summarize-resources.py create mode 100755 scripts/up.sh create mode 100755 scripts/verify.sh create mode 100644 scripts/wait-ready.py create mode 100644 src/__init__.py create mode 100644 src/auth.py create mode 100644 src/chat_repository.py create mode 100644 src/database.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/session_cache.py create mode 100644 src/user_repository.py create mode 100644 student-tests/README.md create mode 100644 student-tests/baseline.py create mode 100644 student-tests/check_auth.py create mode 100644 student-tests/dns.py create mode 100644 student-tests/readiness.py create mode 100644 student-tests/scenarios.py create mode 100644 student-tests/test_event_loop.py diff --git a/.dockerignore b/.dockerignore index bc16e556..f6879b61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store + +DEFENSE.md diff --git a/.env.example b/.env.example index d32b9be0..d0974a53 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,3 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +PUBLIC_PORT=8080 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +# scripts/bootstrap.py creates .env and secret files; no credentials here. diff --git a/.gitignore b/.gitignore index b5a71896..e840fbc8 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +DEFENSE.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bf08a8cc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.11.16-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV APP_HOST=0.0.0.0 +ENV APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . + +RUN python -m pip install --no-cache-dir -r requirements.txt + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser + +COPY --chown=appuser:appuser src/ ./src/ + +COPY --chown=appuser:appuser db/ ./db/ + +COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json + +USER appuser + +EXPOSE 8080 + +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 CMD python -c "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()" + +CMD ["python", "-m", "uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "4", "--timeout-keep-alive", "2"] diff --git a/REPORT.md b/REPORT.md index c2679005..ea5f6c2f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,236 @@ -# Отчёт — лабораторная 3 +# Лабораторная №3 — Балансировка и отказ API-реплики -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Автор: **Дугина Александра, БПИ2301**. Целевая оценка: **4**. Треки на 5 не выполнялись. +Работа выполнена локально 06.11.2026 (Москва; UTC-файлы датированы также 05.11 вечером). +Ветка `solution/BPI2301-Dugina`. Исходный HEAD lab3: `0ab3a341aa7ba32363ce43061d4ff02b1f2964af`. +Коммиты, staging и публикация не выполнялись: изменения оставлены для предварительной проверки. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Desktop с Compose v2, Python 3.11+ и make. На использованном Mac Python хоста — 3.13, контейнера — 3.11.16. +Все команды выполняются из **mtusi2026lab3**, соседняя lab2 не нужна. -## Архитектура и решения +```sh +python3 -m venv .venv +.venv/bin/pip install -r requirements-dev.txt +scripts/up.sh +export BASE_URL="http://localhost:$(sed -n 's/^PUBLIC_PORT=//p' .env)" +curl -i "$BASE_URL/health/ready" +make check +make validate PYTHON=.venv/bin/python +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +scripts/restore.sh +python3 scripts/wait-ready.py +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`scripts/up.sh` вызывает bootstrap, `docker compose --env-file .env config --quiet`, сборку и запуск, ожидает две реплики через LB. +`bootstrap.py` создаёт случайный пароль PostgreSQL, DSN в `secrets/` и `.env`, выбирает свободный порт 8080–8099 только при отсутствии `.env`. Существующие настройки не заменяет. В текущем запуске выбран **127.0.0.1:8080**. При уже созданном `.env` и занятом порте меняют только PUBLIC_PORT своей lab3. Чужие контейнеры не останавливаются. +`secrets/`, `.env` игнорируются Git; локальные файлы создаются с umask 077. Compose secrets монтирует их только в нужные контейнеры. Redis доступен исключительно во внутренней сети и не содержит токены в открытом виде. +`scripts/compose.sh` фиксирует каталог проекта и `.env`, на macOS добавляет Docker credential helper в PATH. +Swagger UI: `http://localhost:8080/docs`; `/openapi.json` отдаёт обязательный контракт без изменения исходного файла. -## Доказательства критериев +Безопасная остановка: `scripts/compose.sh stop`. Повторный запуск: `scripts/up.sh`. +`restart.sh` перезапускает API, PostgreSQL, Redis и LB, не удаляет контейнеры/volumes и не меняет секреты. +Данные живут в `mtusi2026lab3_postgres_data` и `mtusi2026lab3_redis_data`; сеть — `mtusi2026lab3_default`. +Удаление учебных данных — отдельная намеренная административная операция, при выполнении работы не применялась. Обычные start/restart/down без удаления volumes данные не очищают. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Архитектура и версии -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +```text +клиент → 127.0.0.1:8080 → HAProxy → api × 2 (Uvicorn/FastAPI) + ├── PostgreSQL: данные, сессии, идемпотентность + └── Redis: общий кэш профилей сессий +``` -## Эксперименты и отказы +| Компонент | Версия | CPU | RAM | +|---|---|---:|---:| +| API, каждая реплика | Python 3.11.16 slim-bookworm, FastAPI 0.115.12, Uvicorn 0.34.2 | 0,75 | 384 MiB | +| Мигратор | тот же образ приложения, отдельный one-shot процесс | 0,5 | 256 MiB | +| PostgreSQL | 16.8-bookworm | 1 | 512 MiB | +| Redis | 7.4.2-alpine | 0,25 | 128 MiB | +| HAProxy | 3.0.9-alpine | 0,5 | 128 MiB | -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Остальные зависимости закреплены в `requirements.txt`: psycopg 3.2.6, psycopg-pool 3.2.6, redis-py 5.2.1, Argon2-cffi 23.1.0, Pydantic 2.11.4. Образы поддерживают ARM64. Docker Engine 28.0.1, Compose 2.33.1-desktop.1. +Хост: MacBookPro18,3, Apple M1 Pro ARM64, 8 CPU, 16 GiB RAM; Docker VM имеет 8 CPU и 8 218 316 800 байт RAM. Полные условия — `evidence/environment.txt`. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Перенесены собственные `src/`, SQL начальной схемы, зависимости, Dockerfile и тест авторизации из lab2 (`bd34c1050a61d92faa30468eace4e4dadea306c3`, та же рабочая ветка). Lab2 читалась, её файлы и данные не изменялись. Контракты, публичные тесты и исходный README lab3 сохранены. Добавлены балансировка, заголовок процесса, отдельный мигратор, общая идемпотентность, эксплуатационные скрипты, эксперименты. API — один Compose-service с `deploy.replicas: 2`, без `container_name`, без опубликованных портов. Публичен только LB на loopback. -## Ограничения и следующие шаги +HAProxy выбран за активные HTTP readiness-проверки и штатный Docker DNS resolver/server-template; это позволяет масштабировать один сервис без перечня двух фиксированных имён. Round robin распределяет HTTP-запросы без cookie/sticky session. В конфигурации десять слотов backend (поддерживаемый предел данного учебного конфига). Незаполненные слоты переходят в DNS maintenance. Логи ошибок, переходы UP/DOWN и HTTP-статусы пишутся в stdout контейнера. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Состояние, контракт и идемпотентность + +PostgreSQL — источник истины для пользователей, membership, чатов, сообщений и сессий. Пароли — Argon2id: time_cost=3, memory_cost=65536 KiB, parallelism=4. Basic используется только для выдачи случайного токена с 256 битами энтропии. В БД хранится SHA-256 токена и expiry; TTL по умолчанию 3600 секунд, рестарт его не продлевает. На каждом защищённом запросе БД проверяет expiry/revoked_at. Redis кэширует профиль пользователя по хешу токена, но не заменяет проверку сессии: отказ Redis допускает чтение профиля из PostgreSQL. Сессионное состояние не локально. Logout отзывает сессию в PostgreSQL и удаляет кэш. Redis использует AOF с appendfsync always и отдельный volume. + +Доступ к чату проверяется перед операцией; посторонний получает 404. Ошибки соответствуют контракту, request_id в JSON совпадает с X-Request-Id. Каждый созданный API ответ, включая ошибки/health/204, получает X-Instance-Id (случайный UUID при импорте приложения) и X-Request-Id. ID стабилен в жизни одного Uvicorn-процесса, после рестарта меняется. LB-ошибки соединения могут не иметь этих заголовков. + +Пагинация keyset по `(created_at, id)`, limit 1–100; cursor содержит область user/chat и ID последней записи, проверяется принадлежность. История читается непосредственно из PostgreSQL без устаревающего кэша. При конкурентных вставках между страницами нет snapshot: поздний COMMIT с более ранним created_at может оказаться позади уже выданного cursor. Для неизменной истории обход без повторов; это соответствует контракту. Вложения не добавлены: CreateMessage lab3 принимает только text. + +`Idempotency-Key` обязателен, ASCII `[A-Za-z0-9._:-]{1,128}`. Таблица `message_requests` имеет PK `(chat_id, sender_id, key)`; HTTP-метод и тип пути фиксированы единственным POST сообщений. Записи не удаляются: защита дольше 24 часов (плата — рост таблицы). + +В одной транзакции выполняются: +1. INSERT запроса с ON CONFLICT DO NOTHING. +2. Победитель создаёт сообщение и связывает результат с ключом. +3. Конкурирующий INSERT ожидает завершения транзакции победителя. Следующий SELECT в READ COMMITTED видит готовое сообщение. Одинаковый text возвращает тот же объект и 201; отличный — 409. + +COMMIT атомарен для сообщения и результата. При исключении обе записи откатываются; транзиентные ошибки не закрепляются. Другие пользователи/чаты имеют независимые ключи. Аутентификация и доступ проверяются заново до повтора. Участники неизменяемы в обязательном API. Синхронная транзакция отправки вынесена в thread pool, поэтому HTTP-запросы могут реально конкурировать и внутри одного процесса. Локального словаря ключей нет. + +## Готовность, отказ и остановка + +`/health/live` проверяет жизнь процесса. `/health/ready` выполняет SELECT 1 в обязательной PostgreSQL; Redis не критичен благодаря fallback. Если PostgreSQL недоступен — 503 в общем формате. В `evidence/readiness.json` зафиксирован реальный эксперимент: после stop PostgreSQL у обеих API live=200 и ready=503; после stop Redis ready=200 и профиль читается через fallback. Оба хранилища затем восстановлены. Пул: 1–10 соединений на API, ожидание до 5 с, connect_timeout 5 с, statement_timeout 5 с. + +HAProxy проверяет каждую реплику раз в 1 с, при изменении состояния fastinter 500 мс, fall=2, rise=2, timeout check/connect=1 с. Для отказа соединения практический бюджет обнаружения порядка 2–4 с при свободном Docker-host; это расчёт по настройкам, не гарантия при остановке всей VM. Read timeout сервера 8 с, клиента 10 с. Docker-health API — каждые 2 с, timeout 3 с, retries 3, start_period 5 с; сам Docker-health не маршрутизирует запросы. Решение о маршрутизации принимает HAProxy. + +Docker DNS `127.0.0.11:53`, hold valid/obsolete/NX=1 с, resolver retry/timeout=1 с обновляет server-template. Отдельный тест реально изменил IP работающей API-реплики и подтвердил её повторную доступность через LB (`dns-change.json`). Для занятия старого IP кратковременно запускалась третья настоящая API-реплика; после теста она удалена без удаления данных. Проверка включает Docker CLI и ожидание, поэтому её 9,420 с — не чистый DNS latency. + +PID 1 — Uvicorn, STOPSIGNAL SIGTERM. При сигнале сервер закрывает listener, ждёт активные запросы до 4 с; shutdown hook закрывает пул. Compose даёт 6 с до SIGKILL. После закрытия listener HAProxy исключает backend активными проверками. Полного drain перед остановкой нет: краткие ошибки допустимы и измерены. LB не делает автоматический replay POST (`retries 0`): неопределённую запись повторяет клиент с тем же ключом. `stop-one.sh` опрашивает прямой health внутри только API-контейнеров, точно сопоставляет TARGET_INSTANCE_ID и останавливает найденный контейнер; остальные компоненты не трогает. Восстановление — `scripts/restore.sh`, затем ожидание двух ID. + +## Миграции и rolling update + +Миграции не запускаются из API/startup и не подключены к docker-entrypoint PostgreSQL. Сервис migrate ждёт healthy PostgreSQL; API ждут `service_completed_successfully`. SQL-файлы применяются в одной транзакции под `pg_advisory_xact_lock(310026)` и записываются в `schema_migrations`. Это также сериализует случайный повторный запуск runner. Времена применения двух миграций зафиксированы; явный повтор runner вывел Already applied, таблица до/после совпала. Восстановление API не запускает зависимости. Мигратор может вызываться повторно при полном `up`, но **каждая версия схемы применяется ровно один раз**, а не каждым API. + +002 — добавление таблицы без изменения старых колонок, совместимо с работающей API lab2. При демонстрационном обновлении схема не менялась; версии образов старого/нового API записаны в JSON. Для будущих несовместимых миграций этот скрипт не гарантирует rolling: нужен expand/contract, обратная совместимость либо согласованное окно обслуживания. Автоматического rollback схемы нет. + +```sh +scripts/compose.sh build api +# При НОВЫХ совместимых миграциях отдельно, до замены API: +scripts/compose.sh run --rm --no-deps migrate +scripts/rolling-update.sh +# Или эксперимент сразу с 4 writers и проверкой подтверждений: +python3 student-tests/scenarios.py rolling --output evidence/rolling-update-new.json +``` + +Алгоритм: сохранить два исходных контейнера; проверить survivor; остановить/удалить только первый API; `up --no-deps --no-recreate --scale api=2 api` создаёт недостающий контейнер новым образом, не пересоздаёт survivor; дождаться readiness нового и наблюдения обоих ID через LB; затем заменить второй исходный. При ошибке готовности дальнейшая замена прекращается. Номера Compose-контейнеров могут увеличиваться; скрипты не зависят от суффиксов 1/2. Это поочерёдная замена Compose на одном host, не встроенный rolling deploy оркестратора, не автоматический rollback и не нулевые ошибки. + +## Команды и результаты испытаний + +```sh +python3 student-tests/check_auth.py +python3 student-tests/scenarios.py idempotency --output evidence/idempotency.json +python3 student-tests/scenarios.py failover --target-index 0 --duration 15 --output evidence/failover-replica-1.json +python3 student-tests/scenarios.py failover --target-index 1 --duration 15 --output evidence/failover-replica-2.json +python3 student-tests/baseline.py +python3 student-tests/scenarios.py rolling --output evidence/rolling-update.json +python3 student-tests/scenarios.py rolling --output evidence/rolling-continuity.json +python3 student-tests/dns.py +python3 student-tests/readiness.py +# Полный повтор (заменяет соответствующие evidence новыми результатами): +scripts/verify.sh +``` + +Публичные make check, validate, test, test-persistence и test-failover прошли. У persistence сняты sanitized StartedAt/ID всех контейнеров до/после, включая оба хранилища; токен сохранился. Собственный тест: 24 параллельных одинаковых POST обслужены обеими репликами, один Message и одна копия в истории; повтор с другим body — 409, чужой чат — 404, отозванный токен — 401, независимые user/chat scope, отсутствующий/неверный ключ — 400. Старение тестовых ключей до 25 часов не отменило повтор. Общая сессия и чтение подтверждённого сообщения наблюдались на обеих репликах. + +| Эксперимент (4 writers) | Duration, с | Успешные / ошибки | Ошибки, % | Successful RPS | p50/p95/p99, мс | Восстановление, с | Потеря подтверждённых | +|---|---:|---:|---:|---:|---|---:|---:| +| Baseline, public load.py, 30 с | 30,077 | 11112 / 0 | 0 | 369,451 | 9,070 / 15,647 / 63,266 | — | не оценивается этим генератором | +| Отказ первой | 16,683 | 4803 / 366 | 7,0807 | 287,901 | 9,761 / 25,936 / 69,031 | 3,277 | 0 из 4804 | +| Отказ второй после восстановления первой | 16,741 | 4282 / 322 | 6,9939 | 255,782 | 11,185 / 28,477 / 69,385 | 2,879 | 0 из 4283 | +| Rolling новым образом | 19,218 | 4768 / 453 | 8,6765 | 248,103 | 9,335 / 51,021 / 80,478 | см. ниже | 0 из 4769 | +| Дополнительный rolling, тот же образ | 20,325 | 5064 / 270 | 5,0619 | 249,150 | 11,866 / 36,704 / 71,268 | см. ниже | 0 из 5065 | + +В первом эксперименте остановлен container `7d2957677a35…`, alias `d2f38075-ad18-4b40-b025-0b6b75b5deec`; во втором — **другой исходный контейнер** `cdc714ebeb09…`, alias `0fa5e6d8-e471-4f44-992c-42c530dceb1f`. JSON хранит полные IDs и факт Running=false после hook. Первый восстановлен до второго эксперимента; после обоих снова наблюдались две реплики. Чтение полной истории выполнялось при остановленной реплике прежней сессией; подтверждены запись и доступность всех подтверждений. + +Время восстановления измерено от начала action до завершения первого успешного POST, начатого **после возврата** action. Это консервативная оценка с затратами Docker CLI, обе менее 10 с. Длительность собственных сценариев включает завершение worker/сборщика ресурсов, поэтому больше запрошенного 15-секундного интервала. Успешные RPS посчитаны по этому полному интервалу. Между экспериментами данные не очищались. + +Rolling новым образом: обе замены заняли 3,447 и 6,235 с; весь action (с подготовкой) 12,639 с. Поле first_success_after_action=12,666 с означает первый POST **после завершения двух замен**, а не 12-секундный простой! Дополнительный эксперимент специально измерил непрерывность: **3424 успешных запроса начаты во время action**, максимальный промежуток между успешными завершениями **1,000 с**; прежняя сессия и все 5065 подтверждений сохранены. Версии образов первого rolling действительно отличаются, повторный rolling проверяет замену контейнеров тем же образом. + +Методика: synthetic POST text, closed loop, уникальные ключи, 4 потока, без прозрачного retry в измерении throughput. Для отказов 2 с до воздействия, далее нагрузка до ~15 с. Все тестовые объекты синтетические. На baseline уже были данные предшествующих проверок, размер БД в конце — `dataset.txt`. CPU/RAM сняты параллельно, интервалы нерегулярны (docker stats и Docker CLI добавляют задержку); peaks — максимум **из выборок**, не абсолютный пик. Baseline дал 9 выборок; API суммарно до 120,70% CPU / 118,69 MiB, PostgreSQL 41,47% / 53,01 MiB, HAProxy 17,39% / 13,63 MiB, Redis 4,16% / 9,33 MiB. CPU 100% означает одно ядро; API в таблице суммируются. Детали остальных экспериментов — `resources-summary.json`, raw samples — в JSON экспериментов. + +Это единичные небольшие генераторы на одном ноутбуке: coordinated omission, overhead Python/клиента, latency только успешных ответов, нет определения saturation point. Предельная производительность и SLA production не заявляются. Baseline выполнялся до финального изменения Swagger; путь сообщений и транзакций тот же. Краткие 503/обрывы при остановке не скрыты; нужны retry клиента с тем же ключом и более аккуратный drain для уменьшения ошибок. + +## Соответствие всем критериям уровней 3 и 4 + +| Уровень / требование | Проверка / реализация | Фактическое доказательство | +|---|---|---| +| 3: воспроизводимый Compose, 2+ настоящих API | up.sh, capture-state.py, CPU/port metadata | containers.json, containers.txt | +| 3: одна публичная точка, общие БД/сессии | compose.yaml, public smoke и persistence | smoke.txt, persistence.txt | +| 3: одна сессия на разных репликах | scenarios.py idempotency, me и история на двух ID | idempotency.json | +| 3: X-Instance-Id реального процесса | опрос внутри контейнера, сопоставление hook | failover-replica-1.json, failover-replica-2.json | +| 3: новые чтения/записи ≤10 с после отказа любой одной | последовательный отказ каждой, полная история до restore | оба failover-replica JSON: 3,277 / 2,879 с | +| 3: подтверждённые сообщения не потеряны | сравнение всех объектов в полной истории | оба failover-replica JSON, lost_confirmed=0 | +| 3: общая идемпотентность, включая concurrent/replay | PostgreSQL транзакция, PK; 24 запроса на обе реплики | idempotency.json | +| 3: make test | публичный контрактный сценарий | smoke.txt | +| 3: persistence и failover tests | make test-persistence, make test-failover | persistence.txt, failover-public.txt | +| 4: health/readiness, timeout, SIGTERM, исключение backend | Dockerfile, HAProxy checks, настоящие stop | оба failover JSON; lb-events.txt | +| 4: восстановление / DNS backend | restore.sh; реальная смена IP | dns-change.json, instances_after_restore в failover JSON | +| 4: отказ каждой реплики, RPS/errors/recovery | scenarios.py failover с индексами 0/1 | оба failover-replica JSON и TXT | +| 4: CPU/RAM limits, нет обхода LB и container_name | Compose + sanitized inspect | containers.json, resources-summary.json | +| 4: корректное общее состояние/секреты | PostgreSQL authority, Redis AOF, secrets mounts | persistence.txt, idempotency.json, compose.yaml | +| 4: rolling под нагрузкой, рабочий survivor | rolling-update.py, 4 writers, проверка данных | rolling-update.json, rolling-continuity.json | +| 4: согласованные однократные миграции / совместимость | one-shot migrate, advisory lock, version table | migrations-initial.txt, migrations-rerun.txt, migrations-before.txt, migrations-after.txt | +| Подготовка: структура/полная схема OpenAPI | make check / make validate | check.txt, validate.txt | + +Все пути доказательств в таблице относительны `evidence/`. HTTP-тесты не заменяют анализа конфигурации миграций/лимитов: оба вида доказательств приведены вместе. + +## Ограничения, источники и ИИ + +Защищён отказ одного API-процесса. Host Docker Desktop, PostgreSQL, LB, Redis-volume/storage — одиночные компоненты; потеря узла/диска, HA БД, backup/restore и power-loss durability в этой работе не испытывались. Redis не критичен для авторизации, PostgreSQL критичен. Рестарт хранилищ даёт временную недоступность, persistence проверяет последующее сохранение данных, а не нулевой downtime. HTTP допустим здесь только на localhost с synthetic данными; TLS, распределённый секретный менеджер и multi-host не входят в объём. Идемпотентность без GC расходует место; нет удаления пользователей/чатов и динамического membership. Rolling не откатывает неработающий образ автоматически. Активный healthcheck не гарантирует отсутствие каждого запроса в переходный интервал. + +Основные материалы: README.md, COURSE.md, CONTRIBUTING.md, contracts/openapi.json, contracts/README.md и публичные проверки данного skeleton; исходники собственной lab2. Использованы библиотеки из requirements.txt и Docker-образы перечисленных версий. Codex/ИИ использован для переноса, реализации инфраструктуры и идемпотентности, написания собственных сценариев, запуска реальных тестов и подготовки отчёта. Числа взяты из запусков, результаты не моделировались. Публичные тесты/контракты не изменены. Окончательную оценку выставляет преподаватель. + +## Точечные исправления замечаний от 06.11.2026 + +Версия проверки: `fixes-20261106-threadpool-timeout`. Предыдущие таблицы и файлы +экспериментов выше относятся к прежнему коду и сохранены без перезаписи. Все новые +результаты — в `evidence/fixes-20261106/`, дата/параметры/SHA-256 конфигурации и +исходников — `metadata.json`, точный diff — `implementation.patch`. + +В `stop-one.sh` и `rolling-update.py` заменено `docker stop --time 6` на +`docker stop --timeout 6`, прежние таймаут и обработка ошибок сохранены. В новых +выводах остановки/замены предупреждения об устаревшем параметре не наблюдались. + +Аудит async handlers выявил синхронные SQL/Redis и Argon2 в event loop. Login стал +`def` и целиком выполняется FastAPI в worker. В регистрации хеширование и INSERT, +в создании чата проверка сессии и транзакция, в отправке сообщения проверки сессии +и доступа теперь явно ожидаются через `run_in_threadpool`. Атомарный +`insert_message` уже исполнялся в потоке и не изменён. Startup/shutdown ожидают +открытие/закрытие прежнего пула также в потоке; синхронные lifecycle callbacks +Starlette в отличие от sync HTTP endpoints автоматически туда не переносятся. +OpenAPI-файл загружается один раз до готовности API, аварийный logger.error +в middleware также перенесён в поток. Остальные async error handlers и read_json +не выполняют SQL/Redis/Argon2; явных Depends нет. Полный аудит — `async-audit.md`. + +Используется общий механизм Starlette/AnyIO, без нового executor на запрос. +Repository получает и возвращает соединение в рамках одного worker-вызова; +открытые connection/cursor между вызовами не передаются, транзакции не разделены. +Пул, SQL, commit/rollback и ограничения идемпотентности остались прежними. +Регрессионный `student-tests/test_event_loop.py` проверяет thread ID блокирующих +границ через ASGI: **7 failures на прежнем образе, 7 OK на исправленном**, включая +сохранение 503/request_id при исключении из worker. Это дополнено реальными +HTTP/БД-тестами; подмены regression test не применяются к работающему серверу. + +Все обязательные команды повторно прошли: `make check`, `make validate +PYTHON=.venv/bin/python`, `make test BASE_URL=http://localhost:8080`, +`make test-persistence ACTION_SCRIPT=scripts/restart.sh` и +`make test-failover ACTION_SCRIPT=scripts/stop-one.sh` (BASE_URL=тот же localhost). +24 конкурентных повтора через обе реплики снова дали один Message и одну копию +в истории; другие body/scope, сессии и права также проверены. Swagger UI и +семантическое равенство /openapi.json обязательному контракту подтверждены. + +| Новое испытание, 4 writers | Successful RPS | Успехи / ошибки | Доля ошибок | Восстановление | Потери подтверждений | +|---|---:|---:|---:|---:|---:| +| Отказ первой реплики | 159,853 | 2658 / 52 | 1,9188% | 2,327 с | 0 из 2659 | +| Отказ второй после восстановления первой | 181,418 | 2904 / 43 | 1,4591% | 1,792 с | 0 из 2905 | +| Последовательный rolling под нагрузкой | 234,740 | 3637 / 275 | 7,0297% | см. ниже | 0 из 3638 | + +Rolling: 1602 успешных запроса начаты во время action; максимальный промежуток +между успешными завершениями 0,998 с. Полный action двух замен занял 9,272 с, +это не длительность простоя. Обновлялся тот же исправленный образ; контейнеры +реально заменены по одному с готовым survivor. ID/StartedAt HAProxy, PostgreSQL +и Redis одинаковы до/после всех failover/rolling: они не останавливались этими +скриптами (`infrastructure-isolation.txt`). Persistence отдельно перезапустил +API и хранилища, сохранив подтверждённые данные и сессию. + +Дополнительные сопоставимые по команде короткие POST-нагрузки (15 с, concurrency=4, +одинаковые Compose-лимиты, новый synthetic чат на запуск) дали до/после: +493,781 / 436,912 RPS, p50 7,056 / 7,930 мс, p95 11,083 / 12,978 мс, +p99 38,780 / 42,866 мс, ошибок 0 / 0. **Улучшение задержек не подтверждено**: +в этих единичных запусках latency увеличилась. Полная БД росла, cache/VM не +изолировались; это не причинный эксперимент и не оценка предельной мощности. +Исправление подтверждает отсутствие SQL/Argon2 в event loop, а не обещает рост +POST throughput; переключения потоков сами добавляют накладные расходы. + +После тестов обе API, HAProxy, PostgreSQL и Redis healthy; мигратор Exited(0). +Публичные tests/contracts и .gitignore при этих исправлениях не изменялись. +Существующие незакоммиченные файлы сохранены; staging/commit/push/PR не выполнялись. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e0ce81b9 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,91 @@ +name: mtusi2026lab3 +services: + postgres: + image: postgres:16.8-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 1.0 + mem_limit: 512m + restart: unless-stopped + redis: + image: redis:7.4.2-alpine + command: [redis-server, --appendonly, 'yes', --appendfsync, always] + volumes: [redis_data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 20 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + migrate: + build: . + image: mtusi2026lab3-api:local + command: [python, -m, src.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + cpus: 0.5 + mem_limit: 256m + restart: 'no' + api: + build: . + image: mtusi2026lab3-api:local + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + deploy: + replicas: 2 + healthcheck: + test: [CMD, python, -c, 'import urllib.request; urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()'] + interval: 2s + timeout: 3s + retries: 3 + start_period: 5s + stop_grace_period: 6s + cpus: 0.75 + mem_limit: 384m + restart: unless-stopped + lb: + image: haproxy:3.0.9-alpine + ports: ['127.0.0.1:${PUBLIC_PORT:-8080}:8080'] + volumes: ['./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro'] + depends_on: + api: + condition: service_healthy + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, 'http://127.0.0.1:8080/health/ready'] + interval: 5s + timeout: 3s + retries: 3 + cpus: 0.5 + mem_limit: 128m + restart: unless-stopped +volumes: + postgres_data: + redis_data: +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 00000000..f47793f5 --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,51 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name VARCHAR(100) NOT NULL + CHECK (length(btrim(display_name)) > 0), + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL + CHECK (length(btrim(title)) > 0), + created_by UUID NOT NULL REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id), + joined_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL + CHECK (length(text) <= 4000 AND length(btrim(text)) > 0), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + token_hash TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user_id + ON chat_members (user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expires_at + ON sessions (expires_at); diff --git a/db/migrations/002_idempotency.sql b/db/migrations/002_idempotency.sql new file mode 100644 index 00000000..69ddcd33 --- /dev/null +++ b/db/migrations/002_idempotency.sql @@ -0,0 +1,9 @@ +CREATE TABLE message_requests ( + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL REFERENCES users(id), + key VARCHAR(128) NOT NULL, + text TEXT NOT NULL, + message_id UUID REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (chat_id, sender_id, key) +); diff --git a/evidence/README.md b/evidence/README.md index 8df70015..5f027c35 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,43 @@ -# Доказательства +# Фактические доказательства lab3 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Запуски: 06.11.2026 Москва, соответствующие UTC timestamps 05.11.2026 вечером. +Исходный commit, hardware, версии — environment.txt. Новый commit не создавался. +Все пользователи/чаты/сообщения синтетические. Токены/пароли не сохранялись сюда. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +| Файлы | Содержимое / команда | +|---|---| +| check.txt, validate.txt | make check, make validate PYTHON=.venv/bin/python | +| smoke.txt, auth.txt | make test, перенесённый собственный check_auth.py | +| compose-validation.txt | docker compose --env-file .env config --quiet | +| persistence.txt | публичный make test-persistence ACTION_SCRIPT=scripts/restart.sh | +| persistence-containers-before/after.json | реальные StartedAt и IDs всех компонентов, без Env/секретов | +| failover-public.txt | публичный make test-failover ACTION_SCRIPT=scripts/stop-one.sh | +| idempotency.json | 24 concurrent повтора, обе реплики, 409, scope, права, logout, возраст 25h | +| failover-replica-1/2.json и .txt | четыре writers, поочерёдный stop каждой, RPS/errors/recovery, все подтверждения; raw resource samples | +| load-baseline.json, .txt, -resources.json | неизменённый tests/load.py, 30 с, concurrency=4; параллельные Docker stats | +| rolling-update.json и .txt | реальное обновление образа, отдельная замена каждого контейнера под нагрузкой | +| rolling-continuity.json и .txt | повторная замена тем же образом, дополнительно успехи во время action и max gap | +| rolling-preparation.txt | подготовительная замена: ранние ConnectionRefused — штатные пробы до готовности нового API, затем события успешной замены | +| dns-change.json | новая IP реальной API, HAProxy снова направляет в неё запросы, старая сессия/данные сохранены | +| readiness.json | PostgreSQL остановлен: live=200/ready=503; Redis остановлен: ready=200, профиль читается | +| migrations-initial/rerun/before/after.txt | два применения только один раз; runner skip при повторе, журнал не меняется | +| resources-summary.json | sampled peak CPU/RAM по сервисам, API суммированы внутри каждой выборки | +| lb-events.txt, graceful-shutdown.txt | настоящие UP/DOWN HAProxy и graceful завершение Uvicorn | +| containers.json и .txt | итоговые контейнеры, health, limits, port bindings, volume names | +| dataset.txt | итоговые количества объектов и размер БД, без содержимого | +| demo.txt, swagger.txt | проверенные демонстрационные helper-команды и UI/точный OpenAPI | +| git-status.txt, changed-files.txt | локальные незакоммиченные файлы, включая untracked; список не является staging | + +RPS измеряет HTTP-подтверждения, не предельную мощность сервера. Ошибки учитываются; +перцентили только успешных запросов. Ресурсы — нерегулярные выборки docker stats, +максимум из них не означает абсолютный максимум. Собственные эксперименты включают +Docker CLI/завершение сборщика в elapsed. Полная методика и ограничения — REPORT.md. +Для rolling first_success_after_action измеряется после двух замен, а не длительность +простоя; непрерывность отдельно измерена в rolling-continuity.json. + +Повторные сценарии создают новые данные и могут заменить одноимённые evidence. +После повторов таблицы REPORT.md автоматически не пересчитываются. + +В раннем failover-replica-1.txt также есть ConnectionRefused при ожидании старта +восстановленного API: это проба до готовности, не финальный результат проверки. +Итог JSON записан после восстановления двух реплик и всех assertions. diff --git a/evidence/auth.txt b/evidence/auth.txt new file mode 100644 index 00000000..b1c265fb --- /dev/null +++ b/evidence/auth.txt @@ -0,0 +1,26 @@ +1. Registration +Registration status: 201 +Registration headers: {'Content-Length': '144', 'Connection': 'keep-alive', 'Content-Type': 'application/json', 'Date': 'Thu, 05 Nov 2026 21:36:34 GMT', 'Keep-Alive': 'timeout=60', 'Proxy-Connection': 'keep-alive', 'Server': 'uvicorn', 'X-Instance-Id': '6ef424a5-b218-4ae3-8d27-b77e60af053d', 'X-Request-Id': 'e6be1a7b-3a6a-4559-8fb3-798251343744'} +Registration body: {'id': '41b7125f-c793-43a6-bd5b-6a77b5b6216d', 'username': 'test_886143f0d7', 'display_name': 'Test User', 'created_at': '2026-11-05T21:36:35.757609Z'} +PASS: registration: 201 (expected 201) +User ID: 41b7125f-c793-43a6-bd5b-6a77b5b6216d + +2. Basic login +PASS: login: 201 (expected 201) + +3. Access with Bearer token +PASS: authorized request: 200 (expected 200) + +4. Access without token +PASS: missing token: 401 (expected 401) + +5. X-User-Id cannot authorize +PASS: X-User-Id rejected: 401 (expected 401) + +6. Logout +PASS: logout: 204 (expected 204) + +7. Reuse revoked token +PASS: revoked token rejected: 401 (expected 401) + +ALL AUTH CHECKS PASSED diff --git a/evidence/changed-files.txt b/evidence/changed-files.txt new file mode 100644 index 00000000..e48127cf --- /dev/null +++ b/evidence/changed-files.txt @@ -0,0 +1,80 @@ + M .dockerignore + M .env.example + M .gitignore + M REPORT.md + M evidence/README.md +?? Dockerfile +?? compose.yaml +?? db/migrations/001_initial.sql +?? db/migrations/002_idempotency.sql +?? evidence/auth.txt +?? evidence/changed-files.txt +?? evidence/check.txt +?? evidence/compose-validation.txt +?? evidence/containers.json +?? evidence/containers.txt +?? evidence/dataset.txt +?? evidence/defense-ignore.txt +?? evidence/demo.txt +?? evidence/diff-stat.txt +?? evidence/dns-change.json +?? evidence/environment.txt +?? evidence/failover-public.txt +?? evidence/failover-replica-1.json +?? evidence/failover-replica-1.txt +?? evidence/failover-replica-2.json +?? evidence/failover-replica-2.txt +?? evidence/git-status.txt +?? evidence/graceful-shutdown.txt +?? evidence/idempotency.json +?? evidence/lb-events.txt +?? evidence/load-baseline-resources.json +?? evidence/load-baseline.json +?? evidence/load-baseline.txt +?? evidence/local-integrity.txt +?? evidence/migrations-after.txt +?? evidence/migrations-before.txt +?? evidence/migrations-initial.txt +?? evidence/migrations-rerun.txt +?? evidence/persistence-containers-after.json +?? evidence/persistence-containers-before.json +?? evidence/persistence.txt +?? evidence/readiness.json +?? evidence/resources-summary.json +?? evidence/rolling-continuity.json +?? evidence/rolling-continuity.txt +?? evidence/rolling-preparation.txt +?? evidence/rolling-update.json +?? evidence/rolling-update.txt +?? evidence/smoke.txt +?? evidence/swagger.txt +?? evidence/validate.txt +?? infra/haproxy.cfg +?? requirements.txt +?? scripts/bootstrap.py +?? scripts/capture-state.py +?? scripts/compose.sh +?? scripts/demo.py +?? scripts/restart.sh +?? scripts/restore.sh +?? scripts/rolling-update.py +?? scripts/rolling-update.sh +?? scripts/stop-one.sh +?? scripts/summarize-resources.py +?? scripts/up.sh +?? scripts/verify.sh +?? scripts/wait-ready.py +?? src/__init__.py +?? src/auth.py +?? src/chat_repository.py +?? src/database.py +?? src/main.py +?? src/migrate.py +?? src/session_cache.py +?? src/user_repository.py +?? student-tests/README.md +?? student-tests/baseline.py +?? student-tests/check_auth.py +?? student-tests/dns.py +?? student-tests/readiness.py +?? student-tests/scenarios.py diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/compose-validation.txt b/evidence/compose-validation.txt new file mode 100644 index 00000000..e3b9582e --- /dev/null +++ b/evidence/compose-validation.txt @@ -0,0 +1 @@ +docker compose --env-file .env config --quiet: exit 0 diff --git a/evidence/containers.json b/evidence/containers.json new file mode 100644 index 00000000..4c8e6d32 --- /dev/null +++ b/evidence/containers.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T21:46:46.843692+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-7", + "id": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:31.38854975Z", + "FinishedAt": "2026-11-05T21:36:31.285110833Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-8", + "id": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:55.388030137Z", + "FinishedAt": "2026-11-05T21:36:40.833224588Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.511391303Z", + "FinishedAt": "2026-11-05T21:36:25.272422303Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:43:27.117319221Z", + "FinishedAt": "2026-11-05T21:43:21.003478676Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:43:30.266519875Z", + "FinishedAt": "2026-11-05T21:43:29.495873805Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/containers.txt b/evidence/containers.txt new file mode 100644 index 00000000..17dad2e0 --- /dev/null +++ b/evidence/containers.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-7 mtusi2026lab3-api:local "python -m uvicorn s…" api 11 minutes ago Up 10 minutes (healthy) 8080/tcp +mtusi2026lab3-api-8 mtusi2026lab3-api:local "python -m uvicorn s…" api 11 minutes ago Up 9 minutes (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0.9-alpine "docker-entrypoint.s…" lb 21 minutes ago Up 10 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab3-migrate-1 sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb "python -m src.migra…" migrate 21 minutes ago Exited (0) 18 minutes ago +mtusi2026lab3-postgres-1 postgres:16.8-bookworm "docker-entrypoint.s…" postgres 21 minutes ago Up 3 minutes (healthy) 5432/tcp +mtusi2026lab3-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 21 minutes ago Up 3 minutes (healthy) 6379/tcp diff --git a/evidence/dataset.txt b/evidence/dataset.txt new file mode 100644 index 00000000..6e1fa0d9 --- /dev/null +++ b/evidence/dataset.txt @@ -0,0 +1,4 @@ + users | chats | messages | sessions | idempotency_records | database_size +-------+-------+----------+----------+---------------------+--------------- + 51 | 19 | 30293 | 51 | 30293 | 24 MB +(1 row) diff --git a/evidence/defense-ignore.txt b/evidence/defense-ignore.txt new file mode 100644 index 00000000..27c90d6e --- /dev/null +++ b/evidence/defense-ignore.txt @@ -0,0 +1 @@ +.gitignore:34:DEFENSE.md DEFENSE.md diff --git a/evidence/demo.txt b/evidence/demo.txt new file mode 100644 index 00000000..6264bcb2 --- /dev/null +++ b/evidence/demo.txt @@ -0,0 +1,47 @@ +Registered and authenticated synthetic user: 6618e4e4-eea0-497a-9d24-d4cbf1cdf4f3 +{ + "id": "45d10623-d313-4746-b084-4419f9b484e7", + "title": "Демонстрация", + "member_ids": [ + "6618e4e4-eea0-497a-9d24-d4cbf1cdf4f3" + ], + "created_at": "2026-11-05T21:45:03.081945Z" +} +HTTP 201 instance 0e475d26-c97b-4ed3-9897-26d984080e01 +{ + "id": "ffb14134-2179-41f5-9193-d1205eb0dd68", + "chat_id": "45d10623-d313-4746-b084-4419f9b484e7", + "sender_id": "6618e4e4-eea0-497a-9d24-d4cbf1cdf4f3", + "text": "Синтетическое сообщение для демонстрации", + "created_at": "2026-11-05T21:45:03.189108Z" +} +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +HTTP 200 instance 626707c6-9597-4f08-8a7f-2de863db772c same_user True +HTTP 200 instance 0e475d26-c97b-4ed3-9897-26d984080e01 same_user True +Distinct API processes: 2 +PASS: 201 + 201 same message 6387d358-4de2-4901-88c3-1c80409c83ff ; changed body 409 +[ + { + "id": "ffb14134-2179-41f5-9193-d1205eb0dd68", + "chat_id": "45d10623-d313-4746-b084-4419f9b484e7", + "sender_id": "6618e4e4-eea0-497a-9d24-d4cbf1cdf4f3", + "text": "Синтетическое сообщение для демонстрации", + "created_at": "2026-11-05T21:45:03.189108Z" + }, + { + "id": "6387d358-4de2-4901-88c3-1c80409c83ff", + "chat_id": "45d10623-d313-4746-b084-4419f9b484e7", + "sender_id": "6618e4e4-eea0-497a-9d24-d4cbf1cdf4f3", + "text": "idem demonstration", + "created_at": "2026-11-05T21:45:03.378749Z" + } +] diff --git a/evidence/diff-stat.txt b/evidence/diff-stat.txt new file mode 100644 index 00000000..02b4f7d5 --- /dev/null +++ b/evidence/diff-stat.txt @@ -0,0 +1,6 @@ + .dockerignore | 2 + + .env.example | 14 +---- + .gitignore | 2 + + REPORT.md | 179 ++++++++++++++++++++++++++++++++++++++++++++++------- + evidence/README.md | 44 ++++++++++++- + 5 files changed, 203 insertions(+), 38 deletions(-) diff --git a/evidence/dns-change.json b/evidence/dns-change.json new file mode 100644 index 00000000..908f11b5 --- /dev/null +++ b/evidence/dns-change.json @@ -0,0 +1,14 @@ +{ + "result": "PASS", + "container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "instance": "f28dc394-94a3-4be1-b2d8-42acba5c5936", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.7", + "discovery_seconds": 9.42, + "failed_attempts": 1, + "observed_instances": [ + "1321fedc-d572-4bc8-8b7b-e0a2ad5742b5", + "f28dc394-94a3-4be1-b2d8-42acba5c5936" + ], + "existing_session_and_confirmed_message": "PASS" +} diff --git a/evidence/environment.txt b/evidence/environment.txt new file mode 100644 index 00000000..d01615ec --- /dev/null +++ b/evidence/environment.txt @@ -0,0 +1,19 @@ +Recorded 2026-11-05T21:43:25Z (2026-11-06 Europe/Moscow) +Commands: sysctl -n machdep.cpu.brand_string; sysctl -n hw.model hw.memsize hw.ncpu; docker info; docker compose version; python3 --version +CPU: Apple M1 Pro +Model: MacBookPro18,3 +Host RAM bytes: 17179869184 +Host CPU count: 8 +Docker Engine: 28.0.1 +Docker OS: Docker Desktop +Docker architecture: aarch64 +Docker VM CPUs: 8 +Docker VM RAM bytes: 8218316800 +Docker Compose: v2.33.1-desktop.1 +Host Python: 3.13.3 +API Python base image: python:3.11.16-slim-bookworm +Public address: http://localhost:8080 +Compose project: mtusi2026lab3 +Lab3 starting HEAD: 0ab3a341aa7ba32363ce43061d4ff02b1f2964af +Lab2 source HEAD: bd34c1050a61d92faa30468eace4e4dadea306c3 +Branch: solution/BPI2301-Dugina diff --git a/evidence/failover-public.txt b/evidence/failover-public.txt new file mode 100644 index 00000000..3e89bec5 --- /dev/null +++ b/evidence/failover-public.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 1, "first_recovery_seconds": 2.04} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-replica-1.json b/evidence/failover-replica-1.json new file mode 100644 index 00000000..11f1966b --- /dev/null +++ b/evidence/failover-replica-1.json @@ -0,0 +1,305 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "d2f38075-ad18-4b40-b025-0b6b75b5deec" + ], + "duration_seconds": 16.683, + "concurrency": 4, + "successful_requests": 4803, + "errors": 366, + "error_ratio": 0.070807, + "successful_rps": 287.901, + "successful_latency_ms": { + "p50": 9.761, + "p95": 25.936, + "p99": 69.031 + }, + "first_success_after_action_seconds": 3.277, + "action_duration_seconds": 3.206, + "confirmed_checked": 4804, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "target_instance": "d2f38075-ad18-4b40-b025-0b6b75b5deec", + "hook_output": "Stopping API container 7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b instance d2f38075-ad18-4b40-b025-0b6b75b5deec\nFlag --time has been deprecated, use --timeout instead\n7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "target_running_after_hook": false + }, + "resources": [ + { + "offset_seconds": 2.647, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "62.56%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "11.53%", + "MemUsage": "44.29MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "812kB / 711kB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "54.48%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "12.08%", + "MemUsage": "46.38MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "1.63MB / 1.37MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.65%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.06%", + "MemUsage": "10.32MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "1.89MB / 1.89MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "39.24%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.30%", + "MemUsage": "37.36MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "2.18MB / 2.51MB", + "PIDs": "13" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.48%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "362kB / 574kB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 6.306, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "79.15%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.27%", + "MemUsage": "50.96MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "4.15MB / 3.41MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "0.00%", + "MemUsage": "0B / 0B", + "Name": "mtusi2026lab3-api-4", + "NetIO": "0B / 0B", + "PIDs": "0" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.10%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.31%", + "MemUsage": "10.64MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "3.78MB / 3.77MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "25.85%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.10%", + "MemUsage": "31.21MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "4.08MB / 4.82MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.18%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "693kB / 1.13MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.953, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "79.47%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.99%", + "MemUsage": "53.72MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "9.33MB / 7.58MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.40%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.86%", + "MemUsage": "11.34MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "5.62MB / 5.68MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "32.02%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.24%", + "MemUsage": "31.93MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "6.24MB / 7.47MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.75%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "956kB / 1.58MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 13.018, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "77.60%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "14.06%", + "MemUsage": "53.99MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "12.5MB / 10.2MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "8.92%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.75%", + "MemUsage": "11.2MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "7MB / 7.13MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "23.80%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.33%", + "MemUsage": "32.43MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "7.87MB / 9.47MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.39%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.36MB / 2.26MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.683, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.13%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "14.06%", + "MemUsage": "53.97MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "15.4MB / 12.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.64%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.72%", + "MemUsage": "11.16MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "8.22MB / 8.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "21.52%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.41%", + "MemUsage": "32.83MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "9.31MB / 11.2MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.95%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.48%", + "MemUsage": "9.57MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.61MB / 2.69MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "recorded_at_utc": "2026-11-05T21:29:22.573332+00:00" +} diff --git a/evidence/failover-replica-1.txt b/evidence/failover-replica-1.txt new file mode 100644 index 00000000..0d5df3d7 --- /dev/null +++ b/evidence/failover-replica-1.txt @@ -0,0 +1,80 @@ +Traceback (most recent call last): + File "/usr/local/lib/python3.11/urllib/request.py", line 1348, in do_open + h.request(req.get_method(), req.selector, req.data, headers, + File "/usr/local/lib/python3.11/http/client.py", line 1351, in request + self._send_request(method, url, body, headers, encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1397, in _send_request + self.endheaders(body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1346, in endheaders + self._send_output(message_body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1106, in _send_output + self.send(msg) + File "/usr/local/lib/python3.11/http/client.py", line 1044, in send + self.connect() + File "/usr/local/lib/python3.11/http/client.py", line 1010, in connect + self.sock = self._create_connection( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/socket.py", line 863, in create_connection + raise exceptions[0] + File "/usr/local/lib/python3.11/socket.py", line 848, in create_connection + sock.connect(sa) +ConnectionRefusedError: [Errno 111] Connection refused + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "", line 1, in + File "/usr/local/lib/python3.11/urllib/request.py", line 216, in urlopen + return opener.open(url, data, timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 519, in open + response = self._open(req, data) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 536, in _open + result = self._call_chain(self.handle_open, protocol, protocol + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 496, in _call_chain + result = func(*args) + ^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1377, in http_open + return self.do_open(http.client.HTTPConnection, req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1351, in do_open + raise URLError(err) +urllib.error.URLError: +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "d2f38075-ad18-4b40-b025-0b6b75b5deec" + ], + "duration_seconds": 16.683, + "concurrency": 4, + "successful_requests": 4803, + "errors": 366, + "error_ratio": 0.070807, + "successful_rps": 287.901, + "successful_latency_ms": { + "p50": 9.761, + "p95": 25.936, + "p99": 69.031 + }, + "first_success_after_action_seconds": 3.277, + "action_duration_seconds": 3.206, + "confirmed_checked": 4804, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "target_instance": "d2f38075-ad18-4b40-b025-0b6b75b5deec", + "hook_output": "Stopping API container 7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b instance d2f38075-ad18-4b40-b025-0b6b75b5deec\nFlag --time has been deprecated, use --timeout instead\n7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "target_running_after_hook": false + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "recorded_at_utc": "2026-11-05T21:29:22.573332+00:00" +} diff --git a/evidence/failover-replica-2.json b/evidence/failover-replica-2.json new file mode 100644 index 00000000..2b4cd70b --- /dev/null +++ b/evidence/failover-replica-2.json @@ -0,0 +1,305 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "duration_seconds": 16.741, + "concurrency": 4, + "successful_requests": 4282, + "errors": 322, + "error_ratio": 0.069939, + "successful_rps": 255.782, + "successful_latency_ms": { + "p50": 11.185, + "p95": 28.477, + "p99": 69.385 + }, + "first_success_after_action_seconds": 2.879, + "action_duration_seconds": 2.863, + "confirmed_checked": 4283, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "target_instance": "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "hook_output": "Stopping API container cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa instance 0fa5e6d8-e471-4f44-992c-42c530dceb1f\nFlag --time has been deprecated, use --timeout instead\ncdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "target_running_after_hook": false + }, + "resources": [ + { + "offset_seconds": 2.682, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "45.68%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "14.07%", + "MemUsage": "54.02MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "17.9MB / 14.9MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "53.28%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "11.92%", + "MemUsage": "45.78MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "1.35MB / 1.15MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.92%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.99%", + "MemUsage": "11.5MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "10.7MB / 10.9MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 655kB", + "CPUPerc": "33.11%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.92%", + "MemUsage": "40.55MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "10.2MB / 13.3MB", + "PIDs": "13" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.45%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.332MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.85MB / 3.09MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 6.359, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.00%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "0.00%", + "MemUsage": "0B / 0B", + "Name": "mtusi2026lab3-api-3", + "NetIO": "0B / 0B", + "PIDs": "0" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "82.21%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "16.48%", + "MemUsage": "63.28MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "4.52MB / 3.7MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.46%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.09%", + "MemUsage": "11.64MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "12MB / 12.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 23.5MB", + "CPUPerc": "28.34%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.68%", + "MemUsage": "34.21MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "12MB / 15.5MB", + "PIDs": "10" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.83%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.14MB / 3.58MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.38, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "78.06%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.88%", + "MemUsage": "53.31MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "7.46MB / 6.08MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.06%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.20%", + "MemUsage": "11.78MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "13.6MB / 13.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 33.3MB", + "CPUPerc": "21.21%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.28%", + "MemUsage": "37.28MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "14MB / 17.9MB", + "PIDs": "11" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.38%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.4MB / 4.03MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 13.052, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "78.38%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.92%", + "MemUsage": "53.46MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "11.4MB / 9.26MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.79%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.34%", + "MemUsage": "11.96MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "15.3MB / 15.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 40.4MB", + "CPUPerc": "22.89%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.33%", + "MemUsage": "37.52MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "15.4MB / 19.6MB", + "PIDs": "11" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.08%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.75MB / 4.62MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.741, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "5.70%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.93%", + "MemUsage": "53.5MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "14MB / 11.3MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.78%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.37%", + "MemUsage": "12MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "16.4MB / 16.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 49.9MB", + "CPUPerc": "2.07%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.45%", + "MemUsage": "38.15MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "17.3MB / 21.9MB", + "PIDs": "11" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.84%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.32MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.98MB / 5.01MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "recorded_at_utc": "2026-11-05T21:30:26.357776+00:00" +} diff --git a/evidence/failover-replica-2.txt b/evidence/failover-replica-2.txt new file mode 100644 index 00000000..657eec83 --- /dev/null +++ b/evidence/failover-replica-2.txt @@ -0,0 +1,36 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "duration_seconds": 16.741, + "concurrency": 4, + "successful_requests": 4282, + "errors": 322, + "error_ratio": 0.069939, + "successful_rps": 255.782, + "successful_latency_ms": { + "p50": 11.185, + "p95": 28.477, + "p99": 69.385 + }, + "first_success_after_action_seconds": 2.879, + "action_duration_seconds": 2.863, + "confirmed_checked": 4283, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "target_instance": "0fa5e6d8-e471-4f44-992c-42c530dceb1f", + "hook_output": "Stopping API container cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa instance 0fa5e6d8-e471-4f44-992c-42c530dceb1f\nFlag --time has been deprecated, use --timeout instead\ncdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "target_running_after_hook": false + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "recorded_at_utc": "2026-11-05T21:30:26.357776+00:00" +} diff --git a/evidence/fixes-20261106/README.md b/evidence/fixes-20261106/README.md new file mode 100644 index 00000000..000338ae --- /dev/null +++ b/evidence/fixes-20261106/README.md @@ -0,0 +1,45 @@ +# Два точечных исправления — 06.11.2026 (Москва) + +Версия: fixes-20261106-threadpool-timeout. UTC-время запусков — 05.11.2026 вечером. +metadata.json фиксирует SHA-256 кода/конфигурации, параметры нагрузки и время. +Старые доказательства в родительской evidence/ не перезаписывались. + +- implementation.patch: изменения main.py и двух вызовов Docker stop. +- async-audit.md: все async handlers и способ переноса синхронных операций. +- event-loop-before.txt: ожидаемые 7 failures при воспроизведении проблемы на старом образе. +- event-loop-after.txt: тот же регрессионный тест, 7 OK на новом образе. +- build.txt, deploy.txt: сборка и начальная последовательная установка исправленного API. +- check.txt, validate.txt, smoke.txt, persistence.txt, failover-public.txt: обязательные make-проверки после исправления. +- persistence-before/after.json: реальные контейнеры/StartedAt до и после перезапуска. +- idempotency.json/.txt: 24 concurrent POST через обе реплики; одна копия результата, 409/scope/session/access также проверены. +- failover-replica-1/2.json/.txt: фактическая остановка каждой API и восстановление, 4 writers, 15 с, RPS/errors/recovery/подтверждения. +- rolling-update.json/.txt: поочерёдная замена контейнеров тем же исправленным образом под 4 writers; подтверждения и непрерывность. +- faults-before/after.json, infrastructure-isolation.txt: HAProxy/БД/Redis не рестартовали во время отказов API и rolling. +- load-before/after.json/.txt: одинаковые параметры public load.py, 15 с / concurrency=4; по одному запуску, не доказательство улучшения latency/capacity. +- containers-final.json/.txt, swagger.txt: штатное итоговое состояние, Swagger и обязательный OpenAPI. +- git-status.txt, integrity.txt: локальное состояние Git и проверка защищённых файлов. + +Команды выполнялись из корня репозитория с BASE_URL=http://localhost:8080: + +```sh +make check +make validate PYTHON=.venv/bin/python +scripts/compose.sh build api +scripts/rolling-update.sh +docker compose --env-file .env exec -T api python - < student-tests/test_event_loop.py +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +scripts/restore.sh +python3 scripts/wait-ready.py +python3 student-tests/scenarios.py idempotency --output evidence/fixes-20261106/idempotency.json +python3 student-tests/scenarios.py failover --target-index 0 --duration 15 --output evidence/fixes-20261106/failover-replica-1.json +python3 student-tests/scenarios.py failover --target-index 1 --duration 15 --output evidence/fixes-20261106/failover-replica-2.json +python3 student-tests/scenarios.py rolling --output evidence/fixes-20261106/rolling-update.json +python3 tests/load.py --duration 15 --concurrency 4 --output +``` + +Все результаты synthetic, credentials отсутствуют. Время восстановления failover +считается от начала hook до завершения первого успешного запроса, начатого после +возврата hook. first_success_after_action у rolling относится к завершению двух +замен, а не периоду недоступности. Ошибки переходов посчитаны, не скрыты. diff --git a/evidence/fixes-20261106/async-audit.md b/evidence/fixes-20261106/async-audit.md new file mode 100644 index 00000000..13979e0b --- /dev/null +++ b/evidence/fixes-20261106/async-audit.md @@ -0,0 +1,40 @@ +# Аудит блокирующих операций — исправление от 06.11.2026 + +Версия: fixes-20261106-threadpool-timeout. SHA-256 файлов и время запусков — metadata.json. +Репозитории/транзакции/пул, публичные tests/ и contracts/ не изменялись. + +| Обработчик/граница | До исправления | После | +|---|---|---| +| create_user (async) | Argon2 hash + insert_user/SQL в event loop | два await run_in_threadpool, хеш до открытия соединения | +| login | async с find_user_by_username, Argon2 verify, create_session/SQL | def: FastAPI исполняет всю функцию в worker | +| create_chat (async) | проверка сессии SQL/Redis и insert_chat/транзакция в event loop | оба синхронных вызова await run_in_threadpool | +| send_message (async) | get_current_user и get_chat (SQL/Redis) в event loop; insert_message уже в потоке | authentication/access также await run_in_threadpool; атомарный insert_message сохранён | +| startup/shutdown | sync callbacks непосредственно открывали/закрывали pool в lifecycle event loop | async callbacks ожидают run_in_threadpool(open_database/close_database) | +| request_id_middleware | await call_next; при исключении синхронная запись logger.error | запись логгера ожидается через run_in_threadpool | +| framework async /openapi.json → app.openapi | чтение/парсинг файла на каждом запросе | схема загружается при импорте до готовности сервера, HTTP получает готовый dict | +| read_json (async) | асинхронное чтение тела, JSON/Content-Type validation | без изменений: нет синхронного SQL/сети/Argon2 | +| http_error_handler / database_error_handler / validation_error_handler | создание короткого JSONResponse | без изменений: нет внешнего I/O | + +Других async def в src/ нет. Синхронные GET, logout и health/readiness остаются def, +FastAPI запускает их в worker. Явных Depends нет. get_current_user и get_chat — обычные +синхронные функции: прямой вызов из async теперь явно перенесён в поток; вызов из +синхронного endpoint уже находится в worker. Исключения из worker доходят до прежних +обработчиков, что дополнительно проверено для OperationalError/503 и request_id. + +Starlette использует существующий общий механизм AnyIO threads, новый executor на +HTTP-запрос не создаётся. Каждый вызов repository целиком получает соединение из +существующего psycopg_pool, использует курсор/transaction и возвращает соединение +контекстным менеджером в том же worker-вызове. Открытые connection/cursor между +потоками не передаются. Отдельные вызовы могут попасть в разные workers, но между +ними передаются только готовые данные. Транзакция insert_message не разбита. + +Регрессионная проверка проходит через реальную ASGI-маршрутизацию в отдельном Python +процессе контейнера. Только внешние блокирующие границы подменяются spy-функциями, +фиксирующими thread ID; к работающему серверу эти подмены не применяются. До правки +тот же тест дал 7 failures; после — 7 OK. Реальное хранилище и атомарность отдельно +проверены HTTP-тестами и 24 concurrent повторами через обе API. + +Docker: единственные старые вызовы были scripts/stop-one.sh и +scripts/rolling-update.py, заменено --time на --timeout, значение 6 сохранено. +Код возврата и обработка ошибок не менялись. Старые упоминания в исторических +доказательствах/implementation.patch сохранены как история предыдущей версии. diff --git a/evidence/fixes-20261106/build.txt b/evidence/fixes-20261106/build.txt new file mode 100644 index 00000000..246b6fbb --- /dev/null +++ b/evidence/fixes-20261106/build.txt @@ -0,0 +1,58 @@ +Compose now can delegate build to bake for better performances +Just set COMPOSE_BAKE=true +#0 building with "desktop-linux" instance using docker driver + +#1 [api internal] load build definition from Dockerfile +#1 transferring dockerfile: 925B done +#1 DONE 0.0s + +#2 [api internal] load metadata for docker.io/library/python:3.11.16-slim-bookworm +#2 DONE 0.4s + +#3 [api internal] load .dockerignore +#3 transferring context: 208B done +#3 DONE 0.0s + +#4 [api internal] load build context +#4 transferring context: 62.84kB done +#4 DONE 0.0s + +#5 [api 1/8] FROM docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b +#5 resolve docker.io/library/python:3.11.16-slim-bookworm@sha256:a36c24f9cbdf4fd0f52d67f0823eeac19c2028c637cecc392d97f980d4fec56b 6.8s done +#5 DONE 6.8s + +#6 [api 4/8] RUN python -m pip install --no-cache-dir -r requirements.txt +#6 CACHED + +#7 [api 2/8] WORKDIR /app +#7 CACHED + +#8 [api 3/8] COPY requirements.txt . +#8 CACHED + +#9 [api 5/8] RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin appuser +#9 CACHED + +#10 [api 6/8] COPY --chown=appuser:appuser src/ ./src/ +#10 DONE 0.0s + +#11 [api 7/8] COPY --chown=appuser:appuser db/ ./db/ +#11 DONE 0.0s + +#12 [api 8/8] COPY --chown=appuser:appuser contracts/openapi.json ./contracts/openapi.json +#12 DONE 0.0s + +#13 [api] exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:fd12099a79fc1a4eeccb7755efb2561cf59389e3eb9e874aaa7042102b308ebe done +#13 exporting config sha256:b3f3014a82b1e0ced597dc260411c6e262a28a7aa33317be97e06a770af6d9b9 done +#13 exporting attestation manifest sha256:9ce1e740756fa4220b15afff1abaa174f11c5c50cd240e8abb3c265b52ae3d5b done +#13 exporting manifest list sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351 done +#13 naming to docker.io/library/mtusi2026lab3-api:local done +#13 unpacking to docker.io/library/mtusi2026lab3-api:local 0.0s done +#13 DONE 0.1s + +#14 [api] resolving provenance for metadata file +#14 DONE 0.0s + api Built diff --git a/evidence/fixes-20261106/check.txt b/evidence/fixes-20261106/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/fixes-20261106/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/fixes-20261106/containers-final.json b/evidence/fixes-20261106/containers-final.json new file mode 100644 index 00000000..9a4e4818 --- /dev/null +++ b/evidence/fixes-20261106/containers-final.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T22:19:30.567044+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-10", + "id": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:17:58.900805805Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-11", + "id": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:18:03.528023543Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.982249298Z", + "FinishedAt": "2026-11-05T22:16:44.508581257Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.72922259Z", + "FinishedAt": "2026-11-05T22:16:44.596081923Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.682851423Z", + "FinishedAt": "2026-11-05T22:16:44.563222007Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/fixes-20261106/containers-final.txt b/evidence/fixes-20261106/containers-final.txt new file mode 100644 index 00000000..80ed7db0 --- /dev/null +++ b/evidence/fixes-20261106/containers-final.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-10 mtusi2026lab3-api:local "python -m uvicorn s…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-11 mtusi2026lab3-api:local "python -m uvicorn s…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-lb-1 haproxy:3.0.9-alpine "docker-entrypoint.s…" lb 54 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab3-migrate-1 sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb "python -m src.migra…" migrate 54 minutes ago Exited (0) 51 minutes ago +mtusi2026lab3-postgres-1 postgres:16.8-bookworm "docker-entrypoint.s…" postgres 54 minutes ago Up 2 minutes (healthy) 5432/tcp +mtusi2026lab3-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 54 minutes ago Up 2 minutes (healthy) 6379/tcp diff --git a/evidence/fixes-20261106/deploy.txt b/evidence/fixes-20261106/deploy.txt new file mode 100644 index 00000000..ffa0af91 --- /dev/null +++ b/evidence/fixes-20261106/deploy.txt @@ -0,0 +1,2 @@ +{"old_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", "old_instance": "626707c6-9597-4f08-8a7f-2de863db772c", "new_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", "new_instance": "381f7380-3e89-4437-9716-652e85bcf11a", "survivor_container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "old_ip": "172.24.0.6", "new_ip": "172.24.0.6", "replacement_seconds": 4.034} +{"old_container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", "old_instance": "0e475d26-c97b-4ed3-9897-26d984080e01", "new_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", "new_instance": "8641a83a-051c-4737-99f1-8636896695f0", "survivor_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "old_ip": "172.24.0.5", "new_ip": "172.24.0.5", "replacement_seconds": 3.946} diff --git a/evidence/fixes-20261106/event-loop-after.txt b/evidence/fixes-20261106/event-loop-after.txt new file mode 100644 index 00000000..000c9774 --- /dev/null +++ b/evidence/fixes-20261106/event-loop-after.txt @@ -0,0 +1,12 @@ +test_chat_session_and_transaction (__main__.EventLoopTests.test_chat_session_and_transaction) ... ok +test_lifecycle_pool_open_and_close (__main__.EventLoopTests.test_lifecycle_pool_open_and_close) ... ok +test_login_lookup_verify_and_session (__main__.EventLoopTests.test_login_lookup_verify_and_session) ... ok +test_message_session_access_and_atomic_insert (__main__.EventLoopTests.test_message_session_access_and_atomic_insert) ... ok +test_openapi_does_not_read_file_in_request (__main__.EventLoopTests.test_openapi_does_not_read_file_in_request) ... ok +test_registration_hash_and_insert (__main__.EventLoopTests.test_registration_hash_and_insert) ... ok +test_thread_exception_preserves_error_contract (__main__.EventLoopTests.test_thread_exception_preserves_error_contract) ... ok + +---------------------------------------------------------------------- +Ran 7 tests in 0.055s + +OK diff --git a/evidence/fixes-20261106/event-loop-before.txt b/evidence/fixes-20261106/event-loop-before.txt new file mode 100644 index 00000000..a2024869 --- /dev/null +++ b/evidence/fixes-20261106/event-loop-before.txt @@ -0,0 +1,153 @@ +test_chat_session_and_transaction (__main__.EventLoopTests.test_chat_session_and_transaction) ... FAIL +test_lifecycle_pool_open_and_close (__main__.EventLoopTests.test_lifecycle_pool_open_and_close) ... FAIL +test_login_lookup_verify_and_session (__main__.EventLoopTests.test_login_lookup_verify_and_session) ... FAIL +test_message_session_access_and_atomic_insert (__main__.EventLoopTests.test_message_session_access_and_atomic_insert) ... FAIL +test_openapi_does_not_read_file_in_request (__main__.EventLoopTests.test_openapi_does_not_read_file_in_request) ... Unhandled error: GET /openapi.json request_id=d0a4d4c5-7457-4369-a991-11787d1824fb +FAIL +test_registration_hash_and_insert (__main__.EventLoopTests.test_registration_hash_and_insert) ... FAIL +test_thread_exception_preserves_error_contract (__main__.EventLoopTests.test_thread_exception_preserves_error_contract) ... FAIL + +====================================================================== +FAIL: test_chat_session_and_transaction (__main__.EventLoopTests.test_chat_session_and_transaction) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 97, in test_chat_session_and_transaction + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424] : find_session_user_cached blocked the event loop + +====================================================================== +FAIL: test_lifecycle_pool_open_and_close (__main__.EventLoopTests.test_lifecycle_pool_open_and_close) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 112, in test_lifecycle_pool_open_and_close + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424] : open_database blocked the event loop + +====================================================================== +FAIL: test_login_lookup_verify_and_session (__main__.EventLoopTests.test_login_lookup_verify_and_session) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 90, in test_login_lookup_verify_and_session + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424] : find_user_by_username blocked the event loop + +====================================================================== +FAIL: test_message_session_access_and_atomic_insert (__main__.EventLoopTests.test_message_session_access_and_atomic_insert) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 105, in test_message_session_access_and_atomic_insert + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424, 281473871163424] : find_session_user_cached blocked the event loop + +====================================================================== +FAIL: test_openapi_does_not_read_file_in_request (__main__.EventLoopTests.test_openapi_does_not_read_file_in_request) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 117, in test_openapi_does_not_read_file_in_request +AssertionError: 500 != 200 + +====================================================================== +FAIL: test_registration_hash_and_insert (__main__.EventLoopTests.test_registration_hash_and_insert) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 79, in test_registration_hash_and_insert + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424] : hash_password blocked the event loop + +====================================================================== +FAIL: test_thread_exception_preserves_error_contract (__main__.EventLoopTests.test_thread_exception_preserves_error_contract) +---------------------------------------------------------------------- +Traceback (most recent call last): + File "/usr/local/lib/python3.11/unittest/async_case.py", line 90, in _callTestMethod + if self._callMaybeAsync(method) is not None: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/unittest/async_case.py", line 112, in _callMaybeAsync + return self._asyncioRunner.run( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/runners.py", line 118, in run + return self._loop.run_until_complete(task) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/asyncio/base_events.py", line 654, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "", line 128, in test_thread_exception_preserves_error_contract + File "", line 70, in assert_off_loop +AssertionError: 281473871163424 unexpectedly found in [281473871163424] : hash_password blocked the event loop + +---------------------------------------------------------------------- +Ran 7 tests in 0.051s + +FAILED (failures=7) +exit status 1 diff --git a/evidence/fixes-20261106/failover-public.txt b/evidence/fixes-20261106/failover-public.txt new file mode 100644 index 00000000..6d631efd --- /dev/null +++ b/evidence/fixes-20261106/failover-public.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 120, "failed_attempts": 5, "first_recovery_seconds": 2.593} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/fixes-20261106/failover-replica-1.json b/evidence/fixes-20261106/failover-replica-1.json new file mode 100644 index 00000000..cad89331 --- /dev/null +++ b/evidence/fixes-20261106/failover-replica-1.json @@ -0,0 +1,296 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "c1e6d77a-face-4ab3-b030-25f9ae71ff48", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "duration_seconds": 16.628, + "concurrency": 4, + "successful_requests": 2658, + "errors": 52, + "error_ratio": 0.019188, + "successful_rps": 159.853, + "successful_latency_ms": { + "p50": 12.233, + "p95": 61.318, + "p99": 85.003 + }, + "first_success_after_action_seconds": 2.327, + "action_duration_seconds": 1.512, + "successful_requests_started_during_action": 280, + "max_gap_between_successful_writes_seconds": 1.015, + "confirmed_checked": 2659, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "target_instance": "c1e6d77a-face-4ab3-b030-25f9ae71ff48", + "hook_output": "Stopping API container 3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b instance c1e6d77a-face-4ab3-b030-25f9ae71ff48\n3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "target_running_after_hook": false + }, + "resources": [ + { + "offset_seconds": 2.65, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "72.40%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "12.20%", + "MemUsage": "46.87MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "1.44MB / 1.2MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "54.95%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "11.85%", + "MemUsage": "45.5MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "1.95MB / 1.68MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.68%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.01%", + "MemUsage": "10.25MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "1.56MB / 1.61MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 254kB", + "CPUPerc": "35.83%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.13%", + "MemUsage": "36.51MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "1.18MB / 1.37MB", + "PIDs": "12" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.36%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.30%", + "MemUsage": "9.34MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "325kB / 508kB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 6.311, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.35%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "12.29%", + "MemUsage": "47.19MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "1.95MB / 1.62MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.12%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.45%", + "MemUsage": "10.82MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "2.1MB / 2.15MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "0.80%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.20%", + "MemUsage": "31.76MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "2.25MB / 2.65MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.39%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "433kB / 679kB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.323, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "77.93%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "13.06%", + "MemUsage": "50.13MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "3.99MB / 3.3MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.87%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.34%", + "MemUsage": "10.68MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "2.85MB / 2.94MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "14.38%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.71%", + "MemUsage": "34.34MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "3.31MB / 3.92MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.38%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "597kB / 940kB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 12.968, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "79.83%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "13.30%", + "MemUsage": "51.06MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "6.23MB / 5.12MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.63%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "8.46%", + "MemUsage": "10.82MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "3.81MB / 3.95MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "17.16%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.79%", + "MemUsage": "34.78MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "4.45MB / 5.31MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.36%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "745kB / 1.18MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.628, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.98%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "13.25%", + "MemUsage": "50.87MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "7.8MB / 6.41MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.09%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.07%", + "MemUsage": "11.61MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "4.48MB / 4.65MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "3.00%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "6.85%", + "MemUsage": "35.06MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "5.26MB / 6.29MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.57%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "953kB / 1.51MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "recorded_at_utc": "2026-11-05T22:17:28.118074+00:00" +} diff --git a/evidence/fixes-20261106/failover-replica-1.txt b/evidence/fixes-20261106/failover-replica-1.txt new file mode 100644 index 00000000..e50583a3 --- /dev/null +++ b/evidence/fixes-20261106/failover-replica-1.txt @@ -0,0 +1,38 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "c1e6d77a-face-4ab3-b030-25f9ae71ff48", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "duration_seconds": 16.628, + "concurrency": 4, + "successful_requests": 2658, + "errors": 52, + "error_ratio": 0.019188, + "successful_rps": 159.853, + "successful_latency_ms": { + "p50": 12.233, + "p95": 61.318, + "p99": 85.003 + }, + "first_success_after_action_seconds": 2.327, + "action_duration_seconds": 1.512, + "successful_requests_started_during_action": 280, + "max_gap_between_successful_writes_seconds": 1.015, + "confirmed_checked": 2659, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "target_instance": "c1e6d77a-face-4ab3-b030-25f9ae71ff48", + "hook_output": "Stopping API container 3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b instance c1e6d77a-face-4ab3-b030-25f9ae71ff48\n3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "target_running_after_hook": false + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "recorded_at_utc": "2026-11-05T22:17:28.118074+00:00" +} diff --git a/evidence/fixes-20261106/failover-replica-2.json b/evidence/fixes-20261106/failover-replica-2.json new file mode 100644 index 00000000..dfd70b27 --- /dev/null +++ b/evidence/fixes-20261106/failover-replica-2.json @@ -0,0 +1,296 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "duration_seconds": 16.007, + "concurrency": 4, + "successful_requests": 2904, + "errors": 43, + "error_ratio": 0.014591, + "successful_rps": 181.418, + "successful_latency_ms": { + "p50": 12.49, + "p95": 57.989, + "p99": 83.887 + }, + "first_success_after_action_seconds": 1.792, + "action_duration_seconds": 1.036, + "successful_requests_started_during_action": 193, + "max_gap_between_successful_writes_seconds": 0.99, + "confirmed_checked": 2905, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "target_instance": "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919", + "hook_output": "Stopping API container 40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6 instance f9d6e6aa-e9ef-4d45-8dac-d508a21ea919\n40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "target_running_after_hook": false + }, + "resources": [ + { + "offset_seconds": 2.658, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "72.72%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "13.28%", + "MemUsage": "50.98MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "10MB / 8.42MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "72.60%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "12.14%", + "MemUsage": "46.62MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "1.61MB / 1.33MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.56%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.31%", + "MemUsage": "11.92MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "6.56MB / 6.78MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "36.11%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "8.88%", + "MemUsage": "45.47MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "6.97MB / 8.88MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.23%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.30%", + "MemUsage": "9.348MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.12MB / 1.8MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 5.673, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "79.06%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "12.60%", + "MemUsage": "48.4MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "2.74MB / 2.26MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "7.11%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.45%", + "MemUsage": "12.09MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "7.16MB / 7.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "17.60%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.11%", + "MemUsage": "36.41MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "7.65MB / 9.69MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.91%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.30%", + "MemUsage": "9.344MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.37MB / 2.19MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.32, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "82.58%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "16.47%", + "MemUsage": "63.23MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "4.96MB / 4.07MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.63%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.31%", + "MemUsage": "11.91MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "7.85MB / 8.13MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "13.74%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.17%", + "MemUsage": "36.73MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "8.78MB / 11.1MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.48%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.30%", + "MemUsage": "9.344MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.57MB / 2.53MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 12.982, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "80.22%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "13.30%", + "MemUsage": "51.06MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "7.13MB / 5.85MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "6.90%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.44%", + "MemUsage": "12.09MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "9.05MB / 9.38MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 414kB", + "CPUPerc": "11.88%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.21%", + "MemUsage": "36.93MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "9.56MB / 12MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.66%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.332MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.78MB / 2.85MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.007, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "47.29%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "13.36%", + "MemUsage": "51.3MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "8.67MB / 7.11MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "4.32%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.47%", + "MemUsage": "12.12MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "9.71MB / 10.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 446kB", + "CPUPerc": "9.10%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "7.40%", + "MemUsage": "37.88MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "10.7MB / 13.4MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.57%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.336MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "1.92MB / 3.08MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "c1197537-fac7-4da3-8dd0-039b03cabd4f" + ], + "recorded_at_utc": "2026-11-05T22:17:51.866275+00:00" +} diff --git a/evidence/fixes-20261106/failover-replica-2.txt b/evidence/fixes-20261106/failover-replica-2.txt new file mode 100644 index 00000000..56749413 --- /dev/null +++ b/evidence/fixes-20261106/failover-replica-2.txt @@ -0,0 +1,38 @@ +{ + "mode": "failover", + "result": "PASS", + "instances_before": [ + "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919" + ], + "duration_seconds": 16.007, + "concurrency": 4, + "successful_requests": 2904, + "errors": 43, + "error_ratio": 0.014591, + "successful_rps": 181.418, + "successful_latency_ms": { + "p50": 12.49, + "p95": 57.989, + "p99": 83.887 + }, + "first_success_after_action_seconds": 1.792, + "action_duration_seconds": 1.036, + "successful_requests_started_during_action": 193, + "max_gap_between_successful_writes_seconds": 0.99, + "confirmed_checked": 2905, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "target_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "target_instance": "f9d6e6aa-e9ef-4d45-8dac-d508a21ea919", + "hook_output": "Stopping API container 40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6 instance f9d6e6aa-e9ef-4d45-8dac-d508a21ea919\n40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "target_running_after_hook": false + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "instances_after_restore": [ + "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "c1197537-fac7-4da3-8dd0-039b03cabd4f" + ], + "recorded_at_utc": "2026-11-05T22:17:51.866275+00:00" +} diff --git a/evidence/fixes-20261106/faults-after.json b/evidence/fixes-20261106/faults-after.json new file mode 100644 index 00000000..75af06b5 --- /dev/null +++ b/evidence/fixes-20261106/faults-after.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T22:18:10.606823+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-10", + "id": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:17:58.900805805Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-11", + "id": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:18:03.528023543Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.982249298Z", + "FinishedAt": "2026-11-05T22:16:44.508581257Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.72922259Z", + "FinishedAt": "2026-11-05T22:16:44.596081923Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.682851423Z", + "FinishedAt": "2026-11-05T22:16:44.563222007Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/fixes-20261106/faults-before.json b/evidence/fixes-20261106/faults-before.json new file mode 100644 index 00000000..c3ca8f01 --- /dev/null +++ b/evidence/fixes-20261106/faults-before.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T22:16:47.722187+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-10", + "id": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.962599507Z", + "FinishedAt": "2026-11-05T22:16:44.786304715Z" + }, + "health": "starting", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-9", + "id": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.946506715Z", + "FinishedAt": "2026-11-05T22:16:44.790479923Z" + }, + "health": "starting", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.982249298Z", + "FinishedAt": "2026-11-05T22:16:44.508581257Z" + }, + "health": "starting", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.72922259Z", + "FinishedAt": "2026-11-05T22:16:44.596081923Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.682851423Z", + "FinishedAt": "2026-11-05T22:16:44.563222007Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/fixes-20261106/git-status.txt b/evidence/fixes-20261106/git-status.txt new file mode 100644 index 00000000..0362a4e3 --- /dev/null +++ b/evidence/fixes-20261106/git-status.txt @@ -0,0 +1,68 @@ + M .dockerignore + M .env.example + M .gitignore + M REPORT.md + M evidence/README.md +?? Dockerfile +?? compose.yaml +?? db/ +?? evidence/auth.txt +?? evidence/changed-files.txt +?? evidence/check.txt +?? evidence/compose-validation.txt +?? evidence/containers.json +?? evidence/containers.txt +?? evidence/dataset.txt +?? evidence/defense-ignore.txt +?? evidence/demo.txt +?? evidence/diff-stat.txt +?? evidence/dns-change.json +?? evidence/environment.txt +?? evidence/failover-public.txt +?? evidence/failover-replica-1.json +?? evidence/failover-replica-1.txt +?? evidence/failover-replica-2.json +?? evidence/failover-replica-2.txt +?? evidence/fixes-20261106/ +?? evidence/git-status.txt +?? evidence/graceful-shutdown.txt +?? evidence/idempotency.json +?? evidence/lb-events.txt +?? evidence/load-baseline-resources.json +?? evidence/load-baseline.json +?? evidence/load-baseline.txt +?? evidence/local-integrity.txt +?? evidence/migrations-after.txt +?? evidence/migrations-before.txt +?? evidence/migrations-initial.txt +?? evidence/migrations-rerun.txt +?? evidence/persistence-containers-after.json +?? evidence/persistence-containers-before.json +?? evidence/persistence.txt +?? evidence/readiness.json +?? evidence/resources-summary.json +?? evidence/rolling-continuity.json +?? evidence/rolling-continuity.txt +?? evidence/rolling-preparation.txt +?? evidence/rolling-update.json +?? evidence/rolling-update.txt +?? evidence/smoke.txt +?? evidence/swagger.txt +?? evidence/validate.txt +?? infra/haproxy.cfg +?? requirements.txt +?? scripts/bootstrap.py +?? scripts/capture-state.py +?? scripts/compose.sh +?? scripts/demo.py +?? scripts/restart.sh +?? scripts/restore.sh +?? scripts/rolling-update.py +?? scripts/rolling-update.sh +?? scripts/stop-one.sh +?? scripts/summarize-resources.py +?? scripts/up.sh +?? scripts/verify.sh +?? scripts/wait-ready.py +?? src/ +?? student-tests/ diff --git a/evidence/fixes-20261106/idempotency.json b/evidence/fixes-20261106/idempotency.json new file mode 100644 index 00000000..acd98fa7 --- /dev/null +++ b/evidence/fixes-20261106/idempotency.json @@ -0,0 +1,24 @@ +{ + "result": "PASS", + "parallel_requests": 24, + "parallel_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "session_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "reader_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "same_message": true, + "history_copies": 1, + "different_body": 409, + "scope_user_and_chat": "PASS", + "aged_25h_replay": "PASS", + "revoked_replay": 401, + "outsider_replay": 404, + "recorded_at_utc": "2026-11-05T22:16:43.579727+00:00" +} diff --git a/evidence/fixes-20261106/idempotency.txt b/evidence/fixes-20261106/idempotency.txt new file mode 100644 index 00000000..acd98fa7 --- /dev/null +++ b/evidence/fixes-20261106/idempotency.txt @@ -0,0 +1,24 @@ +{ + "result": "PASS", + "parallel_requests": 24, + "parallel_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "session_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "reader_instances": [ + "381f7380-3e89-4437-9716-652e85bcf11a", + "8641a83a-051c-4737-99f1-8636896695f0" + ], + "same_message": true, + "history_copies": 1, + "different_body": 409, + "scope_user_and_chat": "PASS", + "aged_25h_replay": "PASS", + "revoked_replay": 401, + "outsider_replay": 404, + "recorded_at_utc": "2026-11-05T22:16:43.579727+00:00" +} diff --git a/evidence/fixes-20261106/implementation.patch b/evidence/fixes-20261106/implementation.patch new file mode 100644 index 00000000..5dbea073 --- /dev/null +++ b/evidence/fixes-20261106/implementation.patch @@ -0,0 +1,50 @@ +--- before/src/main.py ++++ after/src/main.py +@@ -52 +52,4 @@ +-app.openapi = lambda: json.loads((Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text()) ++OPENAPI_SCHEMA = json.loads( ++ (Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text() ++) ++app.openapi = lambda: OPENAPI_SCHEMA +@@ -91 +94,2 @@ +- logger.error( ++ await run_in_threadpool( ++ logger.error, +@@ -255 +259,2 @@ +- user = insert_user(username, display_name, hash_password(password)) ++ password_hash = await run_in_threadpool(hash_password, password) ++ user = await run_in_threadpool(insert_user, username, display_name, password_hash) +@@ -268 +273 @@ +-async def login(request: Request): ++def login(request: Request): +@@ -335 +340 @@ +- user = get_current_user(request) ++ user = await run_in_threadpool(get_current_user, request) +@@ -367 +372 @@ +- chat = insert_chat(user["id"], title, normalized_ids) ++ chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids) +@@ -428,2 +433,2 @@ +- user = get_current_user(request) +- chat = get_chat(chat_id, request) ++ user = await run_in_threadpool(get_current_user, request) ++ chat = await run_in_threadpool(get_chat, chat_id, request) +@@ -485,2 +490,2 @@ +-def start_database_pool(): +- open_database() ++async def start_database_pool(): ++ await run_in_threadpool(open_database) +@@ -490,2 +495,2 @@ +-def stop_database_pool(): +- close_database() ++async def stop_database_pool(): ++ await run_in_threadpool(close_database) +--- before/scripts/stop-one.sh ++++ after/scripts/stop-one.sh +@@ -15 +15 @@ +-docker stop --time 6 "$found" ++docker stop --timeout 6 "$found" +--- before/scripts/rolling-update.py ++++ after/scripts/rolling-update.py +@@ -44 +44 @@ +- command('docker', 'stop', '--time', '6', old) ++ command('docker', 'stop', '--timeout', '6', old) diff --git a/evidence/fixes-20261106/infrastructure-isolation.txt b/evidence/fixes-20261106/infrastructure-isolation.txt new file mode 100644 index 00000000..ebace83a --- /dev/null +++ b/evidence/fixes-20261106/infrastructure-isolation.txt @@ -0,0 +1,3 @@ +Assertions passed: HAProxy, PostgreSQL and Redis have identical container IDs and StartedAt before/after public failover, failure of each replica, and rolling update. +Each targeted API was actually stopped (Running=false in its experiment JSON). +No deprecated --time warning in captured stop-one hook outputs or rolling/deployment logs. diff --git a/evidence/fixes-20261106/integrity.txt b/evidence/fixes-20261106/integrity.txt new file mode 100644 index 00000000..1e218c2f --- /dev/null +++ b/evidence/fixes-20261106/integrity.txt @@ -0,0 +1,6 @@ +Public tests/contracts unchanged. +Index has no staged changes. +Lab3 HEAD unchanged. +Lab2 clean. +DEFENSE.md absent from index and ignored: +.gitignore:34:DEFENSE.md DEFENSE.md diff --git a/evidence/fixes-20261106/load-after.json b/evidence/fixes-20261106/load-after.json new file mode 100644 index 00000000..c6d71826 --- /dev/null +++ b/evidence/fixes-20261106/load-after.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.031, + "concurrency": 4, + "successful_requests": 6567, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 436.912, + "successful_latency_ms": { + "p50": 7.93, + "p95": 12.978, + "p99": 42.866 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/fixes-20261106/load-after.txt b/evidence/fixes-20261106/load-after.txt new file mode 100644 index 00000000..c6d71826 --- /dev/null +++ b/evidence/fixes-20261106/load-after.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.031, + "concurrency": 4, + "successful_requests": 6567, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 436.912, + "successful_latency_ms": { + "p50": 7.93, + "p95": 12.978, + "p99": 42.866 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/fixes-20261106/load-before.json b/evidence/fixes-20261106/load-before.json new file mode 100644 index 00000000..33bcb567 --- /dev/null +++ b/evidence/fixes-20261106/load-before.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.033, + "concurrency": 4, + "successful_requests": 7423, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 493.781, + "successful_latency_ms": { + "p50": 7.056, + "p95": 11.083, + "p99": 38.78 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/fixes-20261106/load-before.txt b/evidence/fixes-20261106/load-before.txt new file mode 100644 index 00000000..33bcb567 --- /dev/null +++ b/evidence/fixes-20261106/load-before.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.033, + "concurrency": 4, + "successful_requests": 7423, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 493.781, + "successful_latency_ms": { + "p50": 7.056, + "p95": 11.083, + "p99": 38.78 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/fixes-20261106/metadata.json b/evidence/fixes-20261106/metadata.json new file mode 100644 index 00000000..00b61e4e --- /dev/null +++ b/evidence/fixes-20261106/metadata.json @@ -0,0 +1,33 @@ +{ + "recorded_at_utc": "2026-11-05T22:18:46.221142+00:00", + "configuration_version": "fixes-20261106-threadpool-timeout", + "base_url": "http://localhost:8080", + "git_head": "0ab3a341aa7ba32363ce43061d4ff02b1f2964af", + "git_branch": "solution/BPI2301-Dugina", + "sha256": { + "src/main.py": "09354bd51fbd618c5438ecb5025fb5a7f14b050c98eaea7c0461c71c79804078", + "scripts/stop-one.sh": "229e9313709c42bbbcbc0a4576c2c82ef752f008f16e70f134030dd221d7d9be", + "scripts/rolling-update.py": "717c3727b79ebe9b7c54029ddcb25637b839ba14ddce4785c4eb91c16244cb57", + "compose.yaml": "f7132dab84725586112853905da0dcd66edc7d8284e6fc2a4f1c22daf7e76024", + "Dockerfile": "186785fe02e1752caf2f70f5d3b1ab242399f1451e5097e510025e9818fe11d2", + "requirements.txt": "576ba1ec2817e8c5d24aa4d00ed175363f5aeda609881b04cc3572889f6e056b", + "infra/haproxy.cfg": "a336c7983c6d0e09ce9b7ca90d30a28cfdc6d55b2774720296ce60ffc5fc8a15", + "student-tests/test_event_loop.py": "aaec2bf6a27612d6359149135cd5dcee6b1af582f4c3c7d88115435a898a7382" + }, + "source_before_sha256": { + "rolling-update.py": "eb42b39d8de019fa901594d7bade2e05e5e5df83d6862a43551948c1c4af1ec3", + "stop-one.sh": "432e2b1a5430f663f36ce1b90b88c3595098cf8002118dad57577e52eba6bea8", + "main.py": "94eed3807afee905e72fd86cb64e7bd671dc37e100085eedd040f66b9ef21c94" + }, + "load_parameters": { + "command": "python3 tests/load.py --base-url http://localhost:8080 --duration 15 --concurrency 4 --output ", + "duration_seconds": 15, + "concurrency": 4, + "workload": "closed-loop POST messages; each run uses a new synthetic chat; same Compose limits and generator; no retry", + "limitations": "Single run each, growing total database, no isolated VM/cache reset. Not proof of latency or capacity improvement." + }, + "load_result_recorded_at_utc": { + "load-before.json": "2026-11-05T22:14:36.499924+00:00", + "load-after.json": "2026-11-05T22:16:41.122117+00:00" + } +} diff --git a/evidence/fixes-20261106/persistence-after.json b/evidence/fixes-20261106/persistence-after.json new file mode 100644 index 00000000..2681fdca --- /dev/null +++ b/evidence/fixes-20261106/persistence-after.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T22:16:47.535363+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-10", + "id": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.962599507Z", + "FinishedAt": "2026-11-05T22:16:44.786304715Z" + }, + "health": "starting", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-9", + "id": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.946506715Z", + "FinishedAt": "2026-11-05T22:16:44.790479923Z" + }, + "health": "starting", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.982249298Z", + "FinishedAt": "2026-11-05T22:16:44.508581257Z" + }, + "health": "starting", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.72922259Z", + "FinishedAt": "2026-11-05T22:16:44.596081923Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:16:44.682851423Z", + "FinishedAt": "2026-11-05T22:16:44.563222007Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/fixes-20261106/persistence-before.json b/evidence/fixes-20261106/persistence-before.json new file mode 100644 index 00000000..6b29c344 --- /dev/null +++ b/evidence/fixes-20261106/persistence-before.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T22:16:43.776648+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-10", + "id": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:15:31.280879125Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-9", + "id": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T22:15:26.764366804Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.511391303Z", + "FinishedAt": "2026-11-05T21:36:25.272422303Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:43:27.117319221Z", + "FinishedAt": "2026-11-05T21:43:21.003478676Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:43:30.266519875Z", + "FinishedAt": "2026-11-05T21:43:29.495873805Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/fixes-20261106/persistence.txt b/evidence/fixes-20261106/persistence.txt new file mode 100644 index 00000000..933c59c7 --- /dev/null +++ b/evidence/fixes-20261106/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/fixes-20261106/rolling-update.json b/evidence/fixes-20261106/rolling-update.json new file mode 100644 index 00000000..b20d9dff --- /dev/null +++ b/evidence/fixes-20261106/rolling-update.json @@ -0,0 +1,298 @@ +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "c1197537-fac7-4da3-8dd0-039b03cabd4f" + ], + "duration_seconds": 15.494, + "concurrency": 4, + "successful_requests": 3637, + "errors": 275, + "error_ratio": 0.070297, + "successful_rps": 234.74, + "successful_latency_ms": { + "p50": 8.564, + "p95": 58.735, + "p99": 78.026 + }, + "first_success_after_action_seconds": 9.28, + "action_duration_seconds": 9.272, + "successful_requests_started_during_action": 1602, + "max_gap_between_successful_writes_seconds": 0.998, + "confirmed_checked": 3638, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "old_instance": "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "new_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "new_instance": "ecf3693d-484c-45e5-b410-c2d6798075de", + "survivor_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "old_ip": "172.24.0.5", + "new_ip": "172.24.0.5", + "replacement_seconds": 3.692 + }, + { + "old_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "old_instance": "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "new_container": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "new_instance": "4c1159a5-5851-4b71-b5d9-9cc3b102ab52", + "survivor_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 3.258 + } + ] + }, + "resources": [ + { + "offset_seconds": 1.967, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "54.86%", + "Container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "ID": "40c95c32c342", + "MemPerc": "11.74%", + "MemUsage": "45.09MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "891kB / 760kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "74.19%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "13.32%", + "MemUsage": "51.16MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "10.3MB / 8.63MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.68%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.60%", + "MemUsage": "12.29MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "11.2MB / 11.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 688kB", + "CPUPerc": "32.03%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.28%", + "MemUsage": "47.5MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "11.7MB / 15.1MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.35%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.37%", + "MemUsage": "9.434MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.09MB / 3.36MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 8.8, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "67.90%", + "Container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "ID": "b231901fd029", + "MemPerc": "12.01%", + "MemUsage": "46.12MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "1.32MB / 1.11MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "69.37%", + "Container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "ID": "3039c3a93792", + "MemPerc": "13.31%", + "MemUsage": "51.09MiB / 384MiB", + "Name": "mtusi2026lab3-api-9", + "NetIO": "13.1MB / 10.9MB", + "PIDs": "1" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.77%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.77%", + "MemUsage": "12.5MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "13.6MB / 14.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 737kB", + "CPUPerc": "27.47%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.42%", + "MemUsage": "48.22MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "14MB / 17.9MB", + "PIDs": "15" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.58%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.336MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.57MB / 4.14MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 11.82, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "63.32%", + "Container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "ID": "b231901fd029", + "MemPerc": "12.40%", + "MemUsage": "47.6MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "2.26MB / 1.89MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "63.78%", + "Container": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "ID": "00253cf23092", + "MemPerc": "11.67%", + "MemUsage": "44.81MiB / 384MiB", + "Name": "mtusi2026lab3-api-11", + "NetIO": "599kB / 527kB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.81%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.03%", + "MemUsage": "12.84MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "14.4MB / 14.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 754kB", + "CPUPerc": "28.39%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.87%", + "MemUsage": "50.55MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "15.2MB / 19.4MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "2.68%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.31%", + "MemUsage": "9.352MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "2.71MB / 4.37MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 15.493, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "29.10%", + "Container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "ID": "b231901fd029", + "MemPerc": "12.88%", + "MemUsage": "49.46MiB / 384MiB", + "Name": "mtusi2026lab3-api-10", + "NetIO": "4.54MB / 3.73MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "76.31%", + "Container": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "ID": "00253cf23092", + "MemPerc": "12.69%", + "MemUsage": "48.73MiB / 384MiB", + "Name": "mtusi2026lab3-api-11", + "NetIO": "2.71MB / 2.22MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "3.91%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.99%", + "MemUsage": "12.78MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "16.3MB / 16.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 754kB", + "CPUPerc": "10.53%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.07%", + "MemUsage": "51.54MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "17.6MB / 22.2MB", + "PIDs": "16" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.43%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.30%", + "MemUsage": "9.34MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "3.12MB / 5.05MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T22:18:10.415144+00:00" +} diff --git a/evidence/fixes-20261106/rolling-update.txt b/evidence/fixes-20261106/rolling-update.txt new file mode 100644 index 00000000..d6979891 --- /dev/null +++ b/evidence/fixes-20261106/rolling-update.txt @@ -0,0 +1,58 @@ +{"old_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", "old_instance": "4b3377ed-3b9d-453b-8f93-2336d6da70dc", "new_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", "new_instance": "ecf3693d-484c-45e5-b410-c2d6798075de", "survivor_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "old_ip": "172.24.0.5", "new_ip": "172.24.0.5", "replacement_seconds": 3.692} +{"old_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", "old_instance": "c1197537-fac7-4da3-8dd0-039b03cabd4f", "new_container": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", "new_instance": "4c1159a5-5851-4b71-b5d9-9cc3b102ab52", "survivor_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", "old_ip": "172.24.0.4", "new_ip": "172.24.0.4", "replacement_seconds": 3.258} +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "c1197537-fac7-4da3-8dd0-039b03cabd4f" + ], + "duration_seconds": 15.494, + "concurrency": 4, + "successful_requests": 3637, + "errors": 275, + "error_ratio": 0.070297, + "successful_rps": 234.74, + "successful_latency_ms": { + "p50": 8.564, + "p95": 58.735, + "p99": 78.026 + }, + "first_success_after_action_seconds": 9.28, + "action_duration_seconds": 9.272, + "successful_requests_started_during_action": 1602, + "max_gap_between_successful_writes_seconds": 0.998, + "confirmed_checked": 3638, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "40c95c32c342266457de8e5d8d0a7840b444ba6162d0e43e5ef371ec3c42dac6", + "old_instance": "4b3377ed-3b9d-453b-8f93-2336d6da70dc", + "new_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "new_instance": "ecf3693d-484c-45e5-b410-c2d6798075de", + "survivor_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "old_ip": "172.24.0.5", + "new_ip": "172.24.0.5", + "replacement_seconds": 3.692 + }, + { + "old_container": "3039c3a937922dc400f9a8906a78b30cf4e467a3b8adbe218576b2156056df3b", + "old_instance": "c1197537-fac7-4da3-8dd0-039b03cabd4f", + "new_container": "00253cf230927608a1f1390ce3f0c922168eff92d0aed7691759c2073effbc39", + "new_instance": "4c1159a5-5851-4b71-b5d9-9cc3b102ab52", + "survivor_container": "b231901fd0299c40bb8be4f4903d61a4a06d89e2d222a9fd0e9070e4e5fa7125", + "old_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "new_image": "sha256:3d3a985ddaef76e0669885c6c56c7b22d28ae68427c85501f83aa92f76c00351", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 3.258 + } + ] + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T22:18:10.415144+00:00" +} diff --git a/evidence/fixes-20261106/smoke.txt b/evidence/fixes-20261106/smoke.txt new file mode 100644 index 00000000..88517e6f --- /dev/null +++ b/evidence/fixes-20261106/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/fixes-20261106/swagger.txt b/evidence/fixes-20261106/swagger.txt new file mode 100644 index 00000000..d5aff66e --- /dev/null +++ b/evidence/fixes-20261106/swagger.txt @@ -0,0 +1,2 @@ +GET /docs: 200, SwaggerUIBundle present +GET /openapi.json: 200, exact semantic equality with mandatory contracts/openapi.json diff --git a/evidence/fixes-20261106/validate.txt b/evidence/fixes-20261106/validate.txt new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/fixes-20261106/validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/git-status.txt b/evidence/git-status.txt new file mode 100644 index 00000000..b1d48fed --- /dev/null +++ b/evidence/git-status.txt @@ -0,0 +1,67 @@ + M .dockerignore + M .env.example + M .gitignore + M REPORT.md + M evidence/README.md +?? Dockerfile +?? compose.yaml +?? db/ +?? evidence/auth.txt +?? evidence/changed-files.txt +?? evidence/check.txt +?? evidence/compose-validation.txt +?? evidence/containers.json +?? evidence/containers.txt +?? evidence/dataset.txt +?? evidence/defense-ignore.txt +?? evidence/demo.txt +?? evidence/diff-stat.txt +?? evidence/dns-change.json +?? evidence/environment.txt +?? evidence/failover-public.txt +?? evidence/failover-replica-1.json +?? evidence/failover-replica-1.txt +?? evidence/failover-replica-2.json +?? evidence/failover-replica-2.txt +?? evidence/git-status.txt +?? evidence/graceful-shutdown.txt +?? evidence/idempotency.json +?? evidence/lb-events.txt +?? evidence/load-baseline-resources.json +?? evidence/load-baseline.json +?? evidence/load-baseline.txt +?? evidence/local-integrity.txt +?? evidence/migrations-after.txt +?? evidence/migrations-before.txt +?? evidence/migrations-initial.txt +?? evidence/migrations-rerun.txt +?? evidence/persistence-containers-after.json +?? evidence/persistence-containers-before.json +?? evidence/persistence.txt +?? evidence/readiness.json +?? evidence/resources-summary.json +?? evidence/rolling-continuity.json +?? evidence/rolling-continuity.txt +?? evidence/rolling-preparation.txt +?? evidence/rolling-update.json +?? evidence/rolling-update.txt +?? evidence/smoke.txt +?? evidence/swagger.txt +?? evidence/validate.txt +?? infra/haproxy.cfg +?? requirements.txt +?? scripts/bootstrap.py +?? scripts/capture-state.py +?? scripts/compose.sh +?? scripts/demo.py +?? scripts/restart.sh +?? scripts/restore.sh +?? scripts/rolling-update.py +?? scripts/rolling-update.sh +?? scripts/stop-one.sh +?? scripts/summarize-resources.py +?? scripts/up.sh +?? scripts/verify.sh +?? scripts/wait-ready.py +?? src/ +?? student-tests/ diff --git a/evidence/graceful-shutdown.txt b/evidence/graceful-shutdown.txt new file mode 100644 index 00000000..ea3ba5a7 --- /dev/null +++ b/evidence/graceful-shutdown.txt @@ -0,0 +1,22 @@ +api-8 | INFO: Started server process [1] +api-8 | INFO: Waiting for application startup. +api-7 | INFO: Started server process [1] +api-7 | INFO: Waiting for application startup. +api-8 | INFO: Shutting down +api-8 | INFO: Waiting for application shutdown. +api-8 | INFO: Application shutdown complete. +api-8 | INFO: Finished server process [1] +api-8 | INFO: Started server process [1] +api-8 | INFO: Waiting for application startup. +api-8 | INFO: Shutting down +api-8 | INFO: Waiting for application shutdown. +api-8 | INFO: Application shutdown complete. +api-8 | INFO: Finished server process [1] +api-8 | INFO: Started server process [1] +api-8 | INFO: Waiting for application startup. +api-7 | INFO: Shutting down +api-7 | INFO: Waiting for application shutdown. +api-7 | INFO: Application shutdown complete. +api-7 | INFO: Finished server process [1] +api-7 | INFO: Started server process [1] +api-7 | INFO: Waiting for application startup. diff --git a/evidence/idempotency.json b/evidence/idempotency.json new file mode 100644 index 00000000..90b81a97 --- /dev/null +++ b/evidence/idempotency.json @@ -0,0 +1,24 @@ +{ + "result": "PASS", + "parallel_requests": 24, + "parallel_instances": [ + "626707c6-9597-4f08-8a7f-2de863db772c", + "6ef424a5-b218-4ae3-8d27-b77e60af053d" + ], + "session_instances": [ + "626707c6-9597-4f08-8a7f-2de863db772c", + "6ef424a5-b218-4ae3-8d27-b77e60af053d" + ], + "reader_instances": [ + "626707c6-9597-4f08-8a7f-2de863db772c", + "6ef424a5-b218-4ae3-8d27-b77e60af053d" + ], + "same_message": true, + "history_copies": 1, + "different_body": 409, + "scope_user_and_chat": "PASS", + "aged_25h_replay": "PASS", + "revoked_replay": 401, + "outsider_replay": 404, + "recorded_at_utc": "2026-11-05T21:36:38.732963+00:00" +} diff --git a/evidence/lb-events.txt b/evidence/lb-events.txt new file mode 100644 index 00000000..1d1c13d7 --- /dev/null +++ b/evidence/lb-events.txt @@ -0,0 +1,144 @@ +lb-1 | Server replicas/api3 is going DOWN for maintenance (No IP for server ). 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api4 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api5 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api6 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api7 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api3 is going DOWN for maintenance (No IP for server ). 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api4 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api5 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api6 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api7 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api8 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api9 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (8) : Server replicas/api10 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api8 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api9 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api10 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api3 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api4 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api5 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api6 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api7 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api8 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api9 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api10 is going DOWN for maintenance (No IP for server ). 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api3 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api4 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api5 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api6 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api7 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api8 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api9 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api10 is going DOWN for maintenance (No IP for server ). 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 4ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 4ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api2 is DOWN, reason: Layer4 timeout, check duration: 1001ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is DOWN, reason: Layer4 timeout, check duration: 1001ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 was DOWN and now enters maintenance (No IP for server ). +lb-1 | Server replicas/api1 was DOWN and now enters maintenance (No IP for server ). +lb-1 | [WARNING] (7) : Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 was DOWN and now enters maintenance (No IP for server ). +lb-1 | Server replicas/api2 was DOWN and now enters maintenance (No IP for server ). +lb-1 | [WARNING] (7) : Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 was DOWN and now enters maintenance (No IP for server ). +lb-1 | Server replicas/api1 was DOWN and now enters maintenance (No IP for server ). +lb-1 | [WARNING] (7) : Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 was DOWN and now enters maintenance (No IP for server ). +lb-1 | Server replicas/api2 was DOWN and now enters maintenance (No IP for server ). +lb-1 | [WARNING] (7) : Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 2ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 2ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 4 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 4 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api1 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 2ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 2ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api3 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api3 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api3 is DOWN, reason: Layer7 timeout, check duration: 1005ms. 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api3 is DOWN, reason: Layer7 timeout, check duration: 1005ms. 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api3 is UP, reason: Layer7 check passed, code: 200, check duration: 3ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api3 is UP, reason: Layer7 check passed, code: 200, check duration: 3ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer4 connection problem, info: "Connection refused", check duration: 0ms. 9 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api3 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api4 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api5 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api6 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api7 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api8 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api9 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api10 is going DOWN for maintenance (No IP for server ). 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is going DOWN for maintenance (No IP for server ). 8 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api3 is going DOWN for maintenance (No IP for server ). 7 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api4 is going DOWN for maintenance (No IP for server ). 6 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api5 is going DOWN for maintenance (No IP for server ). 5 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api6 is going DOWN for maintenance (No IP for server ). 4 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api7 is going DOWN for maintenance (No IP for server ). 3 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api8 is going DOWN for maintenance (No IP for server ). 2 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api9 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api10 is going DOWN for maintenance (No IP for server ). 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is going DOWN for maintenance (No IP for server ). 1 active and 0 backup servers left. 1 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | Server replicas/api2 ('api') is UP/READY (resolves again). +lb-1 | [WARNING] (7) : Server replicas/api1 is DOWN, reason: Layer7 wrong status, code: 503, info: "Service Unavailable", check duration: 8ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api1 is DOWN, reason: Layer7 wrong status, code: 503, info: "Service Unavailable", check duration: 8ms. 1 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is DOWN, reason: Layer7 timeout, check duration: 1002ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | Server replicas/api2 is DOWN, reason: Layer7 timeout, check duration: 1002ms. 0 active and 0 backup servers left. 0 sessions active, 0 requeued, 0 remaining in queue. +lb-1 | [WARNING] (7) : Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 7ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api2 is UP, reason: Layer7 check passed, code: 200, check duration: 7ms. 1 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. +lb-1 | [WARNING] (7) : Server replicas/api1 is UP, reason: Layer7 check passed, code: 200, check duration: 1ms. 2 active and 0 backup servers online. 0 sessions requeued, 0 total in queue. diff --git a/evidence/load-baseline-resources.json b/evidence/load-baseline-resources.json new file mode 100644 index 00000000..ec3ef484 --- /dev/null +++ b/evidence/load-baseline-resources.json @@ -0,0 +1,542 @@ +[ + { + "offset_seconds": 2.65, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "51.26%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "11.76%", + "MemUsage": "45.17MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "910kB / 787kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "45.20%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.95%", + "MemUsage": "53.56MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "15.2MB / 12.7MB", + "PIDs": "7" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.28%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "9.99%", + "MemUsage": "12.79MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "18.3MB / 18.7MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 54.4MB", + "CPUPerc": "35.53%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.32%", + "MemUsage": "47.73MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "18.3MB / 24MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "1.35%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "3.01MB / 5.06MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 6.344, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "61.19%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "12.93%", + "MemUsage": "49.63MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "3.29MB / 2.68MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.79%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.96%", + "MemUsage": "53.61MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "18.3MB / 15.1MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.84%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.14%", + "MemUsage": "12.98MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "20.3MB / 20.8MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 66.9MB", + "CPUPerc": "32.17%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.53%", + "MemUsage": "48.8MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "20.7MB / 27MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "4.02%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "3.46MB / 5.82MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.366, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "52.13%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.56%", + "MemUsage": "52.08MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "5.19MB / 4.19MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.93%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.97%", + "MemUsage": "53.63MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "20.2MB / 16.6MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.46%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.40%", + "MemUsage": "13.31MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "21.9MB / 22.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 76.9MB", + "CPUPerc": "37.64%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.60%", + "MemUsage": "49.16MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "22.6MB / 29.4MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.58%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "3.91MB / 6.59MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 13.047, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "57.10%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "16.94%", + "MemUsage": "65.03MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "7.56MB / 6.07MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "55.11%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.97%", + "MemUsage": "53.66MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "22.6MB / 18.5MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "12.95%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.27%", + "MemUsage": "13.15MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "23.9MB / 24.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 89.3MB", + "CPUPerc": "29.40%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.74%", + "MemUsage": "49.86MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "24.9MB / 32.3MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "4.11%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "4.24MB / 7.14MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.738, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "55.93%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.64%", + "MemUsage": "52.37MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "10.1MB / 8.07MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "64.77%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "16.98%", + "MemUsage": "65.21MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "25.1MB / 20.5MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.82%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.40%", + "MemUsage": "13.32MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "25.4MB / 26.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 102MB", + "CPUPerc": "39.07%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.87%", + "MemUsage": "50.51MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "27.5MB / 35.5MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.86%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "4.79MB / 8.09MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 19.761, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "56.03%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "16.09%", + "MemUsage": "61.8MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "12MB / 9.59MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "63.68%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.97%", + "MemUsage": "53.66MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "27MB / 22.1MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.35%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.65%", + "MemUsage": "13.63MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "27.6MB / 28.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 113MB", + "CPUPerc": "35.26%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.97%", + "MemUsage": "51.07MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "29.4MB / 37.9MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.61%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "5.14MB / 8.68MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 23.453, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "57.98%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.59%", + "MemUsage": "52.17MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "14.5MB / 11.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "49.27%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.97%", + "MemUsage": "53.65MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "28.8MB / 23.5MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "17.39%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.46%", + "MemUsage": "13.39MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "29.7MB / 30.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 126MB", + "CPUPerc": "41.47%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.17%", + "MemUsage": "52.09MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "31.9MB / 41MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "4.16%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "5.6MB / 9.45MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 27.135, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "66.27%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.59%", + "MemUsage": "52.18MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "16.8MB / 13.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.40%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.97%", + "MemUsage": "53.63MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "31.8MB / 25.9MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.43%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.52%", + "MemUsage": "13.46MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "31.7MB / 32.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 138MB", + "CPUPerc": "34.11%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.30%", + "MemUsage": "52.73MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "34.2MB / 43.9MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.59%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "5.9MB / 9.97MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 30.151, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "55.92%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.59%", + "MemUsage": "52.19MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "18.8MB / 15MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "55.08%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "14.03%", + "MemUsage": "53.86MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "33.8MB / 27.5MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "16.93%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.59%", + "MemUsage": "13.55MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "33.4MB / 34.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 148MB", + "CPUPerc": "40.51%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.35%", + "MemUsage": "53.01MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "36.2MB / 46.4MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.99%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "6.37MB / 10.8MB", + "PIDs": "6" + } + ] + } +] diff --git a/evidence/load-baseline.json b/evidence/load-baseline.json new file mode 100644 index 00000000..abf78226 --- /dev/null +++ b/evidence/load-baseline.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.077, + "concurrency": 4, + "successful_requests": 11112, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 369.451, + "successful_latency_ms": { + "p50": 9.07, + "p95": 15.647, + "p99": 63.266 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-baseline.txt b/evidence/load-baseline.txt new file mode 100644 index 00000000..abf78226 --- /dev/null +++ b/evidence/load-baseline.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.077, + "concurrency": 4, + "successful_requests": 11112, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 369.451, + "successful_latency_ms": { + "p50": 9.07, + "p95": 15.647, + "p99": 63.266 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/local-integrity.txt b/evidence/local-integrity.txt new file mode 100644 index 00000000..904f080d --- /dev/null +++ b/evidence/local-integrity.txt @@ -0,0 +1,6 @@ +Local generated database password/DSN and demo token/password absent from publishable files. +Public tests/contracts unchanged. +DEFENSE.md absent from Git index. +Git index unchanged, no staged changes. +Lab2 worktree clean. +Lab3 HEAD unchanged: 0ab3a341aa7ba32363ce43061d4ff02b1f2964af diff --git a/evidence/migrations-after.txt b/evidence/migrations-after.txt new file mode 100644 index 00000000..f887c17c --- /dev/null +++ b/evidence/migrations-after.txt @@ -0,0 +1,5 @@ + name | applied_at +---------------------+------------------------------- + 001_initial.sql | 2026-11-05 21:25:04.491797+00 + 002_idempotency.sql | 2026-11-05 21:25:04.491797+00 +(2 rows) diff --git a/evidence/migrations-before.txt b/evidence/migrations-before.txt new file mode 100644 index 00000000..f887c17c --- /dev/null +++ b/evidence/migrations-before.txt @@ -0,0 +1,5 @@ + name | applied_at +---------------------+------------------------------- + 001_initial.sql | 2026-11-05 21:25:04.491797+00 + 002_idempotency.sql | 2026-11-05 21:25:04.491797+00 +(2 rows) diff --git a/evidence/migrations-initial.txt b/evidence/migrations-initial.txt new file mode 100644 index 00000000..76e9e520 --- /dev/null +++ b/evidence/migrations-initial.txt @@ -0,0 +1,4 @@ +migrate-1 | Applied 001_initial.sql +migrate-1 | Applied 002_idempotency.sql +migrate-1 | Already applied 001_initial.sql +migrate-1 | Already applied 002_idempotency.sql diff --git a/evidence/migrations-rerun.txt b/evidence/migrations-rerun.txt new file mode 100644 index 00000000..d5f11201 --- /dev/null +++ b/evidence/migrations-rerun.txt @@ -0,0 +1,2 @@ +Already applied 001_initial.sql +Already applied 002_idempotency.sql diff --git a/evidence/persistence-containers-after.json b/evidence/persistence-containers-after.json new file mode 100644 index 00000000..35fe1bf1 --- /dev/null +++ b/evidence/persistence-containers-after.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T21:36:34.488374+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-7", + "id": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:31.38854975Z", + "FinishedAt": "2026-11-05T21:36:31.285110833Z" + }, + "health": "starting", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-8", + "id": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.667618387Z", + "FinishedAt": "2026-11-05T21:36:25.582363053Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.511391303Z", + "FinishedAt": "2026-11-05T21:36:25.272422303Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.450599553Z", + "FinishedAt": "2026-11-05T21:36:25.298179595Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:36:25.442441512Z", + "FinishedAt": "2026-11-05T21:36:25.292179261Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/persistence-containers-before.json b/evidence/persistence-containers-before.json new file mode 100644 index 00000000..ab605db1 --- /dev/null +++ b/evidence/persistence-containers-before.json @@ -0,0 +1,118 @@ +{ + "recorded_at_utc": "2026-11-05T21:36:24.321295+00:00", + "containers": [ + { + "name": "/mtusi2026lab3-api-7", + "id": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:35:25.733666345Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-api-8", + "id": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:35:33.690575793Z", + "FinishedAt": "0001-01-01T00:00:00Z" + }, + "health": "healthy", + "memory_limit_bytes": 402653184, + "nano_cpus": 750000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-lb-1", + "id": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "image": "sha256:53b93733b922428a1e92baa1a791a7026f8c54bdf6f44f740f124502cfef5b81", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:15.140463173Z", + "FinishedAt": "2026-11-05T21:27:14.924901798Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 500000000, + "port_bindings": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-migrate-1", + "id": "37a1664e149aa281ac1ac1d3e66b497869168d9413fbe514173744f8c20f0deb", + "image": "sha256:06b901465d03fb217cca9c39d62727bb1ee76f9362cfa4bfa6e35f67c71aeabb", + "state": { + "Status": "exited", + "Running": false, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:50.053204384Z", + "FinishedAt": "2026-11-05T21:27:50.744764343Z" + }, + "health": "unhealthy", + "memory_limit_bytes": 268435456, + "nano_cpus": 500000000, + "port_bindings": {}, + "volumes": [] + }, + { + "name": "/mtusi2026lab3-postgres-1", + "id": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "image": "sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:15.114301382Z", + "FinishedAt": "2026-11-05T21:27:14.96051909Z" + }, + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus": 1000000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_postgres_data" + ] + }, + { + "name": "/mtusi2026lab3-redis-1", + "id": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "image": "sha256:02419de7eddf55aa5bcf49efb74e88fa8d931b4d77c07eff8a6b2144472b6952", + "state": { + "Status": "running", + "Running": true, + "ExitCode": 0, + "StartedAt": "2026-11-05T21:27:15.092425215Z", + "FinishedAt": "2026-11-05T21:27:14.952502007Z" + }, + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus": 250000000, + "port_bindings": {}, + "volumes": [ + "mtusi2026lab3_redis_data" + ] + } + ] +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..933c59c7 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/readiness.json b/evidence/readiness.json new file mode 100644 index 00000000..6f94fb9d --- /dev/null +++ b/evidence/readiness.json @@ -0,0 +1,26 @@ +{ + "postgres_stopped": [ + { + "container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "live": 200, + "ready": 503 + }, + { + "container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "live": 200, + "ready": 503 + } + ], + "redis_stopped": [ + { + "container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "ready": 200 + }, + { + "container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "ready": 200 + } + ], + "redis_optional_profile_fallback": 200, + "result": "PASS" +} diff --git a/evidence/resources-summary.json b/evidence/resources-summary.json new file mode 100644 index 00000000..f497dcb8 --- /dev/null +++ b/evidence/resources-summary.json @@ -0,0 +1,107 @@ +{ + "load-baseline": { + "samples": 9, + "service_totals": { + "api": { + "peak_sampled_cpu_percent": 120.7, + "peak_sampled_memory_mib": 118.69 + }, + "lb": { + "peak_sampled_cpu_percent": 17.39, + "peak_sampled_memory_mib": 13.63 + }, + "postgres": { + "peak_sampled_cpu_percent": 41.47, + "peak_sampled_memory_mib": 53.01 + }, + "redis": { + "peak_sampled_cpu_percent": 4.16, + "peak_sampled_memory_mib": 9.33 + } + } + }, + "failover-replica-1": { + "samples": 5, + "service_totals": { + "api": { + "peak_sampled_cpu_percent": 117.04, + "peak_sampled_memory_mib": 90.67 + }, + "lb": { + "peak_sampled_cpu_percent": 15.65, + "peak_sampled_memory_mib": 11.34 + }, + "postgres": { + "peak_sampled_cpu_percent": 39.24, + "peak_sampled_memory_mib": 37.36 + }, + "redis": { + "peak_sampled_cpu_percent": 3.48, + "peak_sampled_memory_mib": 9.57 + } + } + }, + "failover-replica-2": { + "samples": 5, + "service_totals": { + "api": { + "peak_sampled_cpu_percent": 98.96, + "peak_sampled_memory_mib": 99.8 + }, + "lb": { + "peak_sampled_cpu_percent": 12.92, + "peak_sampled_memory_mib": 12.0 + }, + "postgres": { + "peak_sampled_cpu_percent": 33.11, + "peak_sampled_memory_mib": 40.55 + }, + "redis": { + "peak_sampled_cpu_percent": 3.45, + "peak_sampled_memory_mib": 9.33 + } + } + }, + "rolling-update": { + "samples": 4, + "service_totals": { + "api": { + "peak_sampled_cpu_percent": 113.46, + "peak_sampled_memory_mib": 111.97 + }, + "lb": { + "peak_sampled_cpu_percent": 15.15, + "peak_sampled_memory_mib": 14.27 + }, + "postgres": { + "peak_sampled_cpu_percent": 39.75, + "peak_sampled_memory_mib": 58.68 + }, + "redis": { + "peak_sampled_cpu_percent": 4.05, + "peak_sampled_memory_mib": 9.57 + } + } + }, + "rolling-continuity": { + "samples": 6, + "service_totals": { + "api": { + "peak_sampled_cpu_percent": 119.6, + "peak_sampled_memory_mib": 115.39 + }, + "lb": { + "peak_sampled_cpu_percent": 13.62, + "peak_sampled_memory_mib": 14.07 + }, + "postgres": { + "peak_sampled_cpu_percent": 29.51, + "peak_sampled_memory_mib": 60.03 + }, + "redis": { + "peak_sampled_cpu_percent": 2.86, + "peak_sampled_memory_mib": 9.57 + } + } + } +} diff --git a/evidence/rolling-continuity.json b/evidence/rolling-continuity.json new file mode 100644 index 00000000..3aa15048 --- /dev/null +++ b/evidence/rolling-continuity.json @@ -0,0 +1,407 @@ +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "c21667a5-8290-4edd-8df5-15414b3ee461", + "d58f5083-f03d-4e7a-ba3c-6f190661975d" + ], + "duration_seconds": 20.325, + "concurrency": 4, + "successful_requests": 5064, + "errors": 270, + "error_ratio": 0.050619, + "successful_rps": 249.15, + "successful_latency_ms": { + "p50": 11.866, + "p95": 36.704, + "p99": 71.268 + }, + "first_success_after_action_seconds": 15.316, + "action_duration_seconds": 15.308, + "successful_requests_started_during_action": 3424, + "max_gap_between_successful_writes_seconds": 1.0, + "confirmed_checked": 5065, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "old_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", + "new_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "new_instance": "f28dc394-94a3-4be1-b2d8-42acba5c5936", + "survivor_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 8.891 + }, + { + "old_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "old_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", + "new_container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "new_instance": "1321fedc-d572-4bc8-8b7b-e0a2ad5742b5", + "survivor_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.6", + "new_ip": "172.24.0.6", + "replacement_seconds": 4.056 + } + ] + }, + "resources": [ + { + "offset_seconds": 2.69, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "59.19%", + "Container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "ID": "7a804e29765b", + "MemPerc": "13.77%", + "MemUsage": "52.87MiB / 384MiB", + "Name": "mtusi2026lab3-api-5", + "NetIO": "9.04MB / 7.59MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "58.89%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "12.91%", + "MemUsage": "49.58MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "4.6MB / 4MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "11.66%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.56%", + "MemUsage": "13.51MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "44.3MB / 45.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 236MB", + "CPUPerc": "29.51%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "11.71%", + "MemUsage": "59.95MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "47.3MB / 60.8MB", + "PIDs": "16" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.86%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "8.27MB / 13.9MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 5.697, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.50%", + "Container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "ID": "7a804e29765b", + "MemPerc": "13.76%", + "MemUsage": "52.84MiB / 384MiB", + "Name": "mtusi2026lab3-api-5", + "NetIO": "10.2MB / 8.53MB", + "PIDs": "8" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "72.55%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "13.05%", + "MemUsage": "50.13MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "6.53MB / 5.57MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.35%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.78%", + "MemUsage": "13.8MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "45.5MB / 46.7MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 242MB", + "CPUPerc": "25.95%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "11.72%", + "MemUsage": "60.03MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "48.6MB / 62.4MB", + "PIDs": "16" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.56%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "8.49MB / 14.3MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.377, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "77.49%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "13.33%", + "MemUsage": "51.19MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "8.77MB / 7.37MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "42.11%", + "Container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "ID": "42edda539ff1", + "MemPerc": "13.52%", + "MemUsage": "51.93MiB / 384MiB", + "Name": "mtusi2026lab3-api-7", + "NetIO": "41.2kB / 44.7kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.15%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.58%", + "MemUsage": "13.54MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "46.8MB / 48.1MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 250MB", + "CPUPerc": "23.13%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.79%", + "MemUsage": "55.27MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "50.2MB / 64.3MB", + "PIDs": "14" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.29%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "8.77MB / 14.8MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 13.057, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "51.55%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "13.74%", + "MemUsage": "52.77MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "10.9MB / 9.11MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "63.03%", + "Container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "ID": "42edda539ff1", + "MemPerc": "16.31%", + "MemUsage": "62.62MiB / 384MiB", + "Name": "mtusi2026lab3-api-7", + "NetIO": "1.33MB / 1.1MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.38%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.63%", + "MemUsage": "13.61MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "47.9MB / 49.2MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 258MB", + "CPUPerc": "19.84%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "11.42%", + "MemUsage": "58.49MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "51.9MB / 66.3MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.44%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.48%", + "MemUsage": "9.574MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "9.05MB / 15.3MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.066, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "62.85%", + "Container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "ID": "42edda539ff1", + "MemPerc": "12.29%", + "MemUsage": "47.19MiB / 384MiB", + "Name": "mtusi2026lab3-api-7", + "NetIO": "3.15MB / 2.58MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "9.07%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.99%", + "MemUsage": "14.07MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "49.1MB / 50.4MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 264MB", + "CPUPerc": "24.05%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "9.54%", + "MemUsage": "48.83MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "52.8MB / 67.4MB", + "PIDs": "11" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.53%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.32MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "9.21MB / 15.5MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 19.747, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "54.87%", + "Container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "ID": "42edda539ff1", + "MemPerc": "12.94%", + "MemUsage": "49.69MiB / 384MiB", + "Name": "mtusi2026lab3-api-7", + "NetIO": "5.49MB / 4.48MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "47.85%", + "Container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "ID": "21e42c73756f", + "MemPerc": "11.62%", + "MemUsage": "44.64MiB / 384MiB", + "Name": "mtusi2026lab3-api-8", + "NetIO": "918kB / 764kB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "13.62%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.67%", + "MemUsage": "13.65MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "50.2MB / 51.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 273MB", + "CPUPerc": "26.79%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "11.16%", + "MemUsage": "57.14MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "54.8MB / 69.8MB", + "PIDs": "14" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.86%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "9.55MB / 16.1MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T21:35:40.340986+00:00" +} diff --git a/evidence/rolling-continuity.txt b/evidence/rolling-continuity.txt new file mode 100644 index 00000000..38fb9f29 --- /dev/null +++ b/evidence/rolling-continuity.txt @@ -0,0 +1,58 @@ +{"old_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", "old_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", "new_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", "new_instance": "f28dc394-94a3-4be1-b2d8-42acba5c5936", "survivor_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "old_ip": "172.24.0.4", "new_ip": "172.24.0.4", "replacement_seconds": 8.891} +{"old_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", "old_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", "new_container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", "new_instance": "1321fedc-d572-4bc8-8b7b-e0a2ad5742b5", "survivor_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "old_ip": "172.24.0.6", "new_ip": "172.24.0.6", "replacement_seconds": 4.056} +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "c21667a5-8290-4edd-8df5-15414b3ee461", + "d58f5083-f03d-4e7a-ba3c-6f190661975d" + ], + "duration_seconds": 20.325, + "concurrency": 4, + "successful_requests": 5064, + "errors": 270, + "error_ratio": 0.050619, + "successful_rps": 249.15, + "successful_latency_ms": { + "p50": 11.866, + "p95": 36.704, + "p99": 71.268 + }, + "first_success_after_action_seconds": 15.316, + "action_duration_seconds": 15.308, + "successful_requests_started_during_action": 3424, + "max_gap_between_successful_writes_seconds": 1.0, + "confirmed_checked": 5065, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "old_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", + "new_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "new_instance": "f28dc394-94a3-4be1-b2d8-42acba5c5936", + "survivor_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 8.891 + }, + { + "old_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "old_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", + "new_container": "21e42c73756f8b1f7d6942afbc0e276226303d6d6841c33bddc1773c380b5f18", + "new_instance": "1321fedc-d572-4bc8-8b7b-e0a2ad5742b5", + "survivor_container": "42edda539ff17db5619bcb152611ceed41b2f2e40f51a1447e40ace6afbb7cfa", + "old_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.6", + "new_ip": "172.24.0.6", + "replacement_seconds": 4.056 + } + ] + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T21:35:40.340986+00:00" +} diff --git a/evidence/rolling-preparation.txt b/evidence/rolling-preparation.txt new file mode 100644 index 00000000..1ac58606 --- /dev/null +++ b/evidence/rolling-preparation.txt @@ -0,0 +1,100 @@ + Container mtusi2026lab3-api-2 Running + Container mtusi2026lab3-api-3 Creating + Container mtusi2026lab3-api-3 Created + Container mtusi2026lab3-api-3 Starting + Container mtusi2026lab3-api-3 Started +Traceback (most recent call last): + File "/usr/local/lib/python3.11/urllib/request.py", line 1348, in do_open + h.request(req.get_method(), req.selector, req.data, headers, + File "/usr/local/lib/python3.11/http/client.py", line 1351, in request + self._send_request(method, url, body, headers, encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1397, in _send_request + self.endheaders(body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1346, in endheaders + self._send_output(message_body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1106, in _send_output + self.send(msg) + File "/usr/local/lib/python3.11/http/client.py", line 1044, in send + self.connect() + File "/usr/local/lib/python3.11/http/client.py", line 1010, in connect + self.sock = self._create_connection( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/socket.py", line 863, in create_connection + raise exceptions[0] + File "/usr/local/lib/python3.11/socket.py", line 848, in create_connection + sock.connect(sa) +ConnectionRefusedError: [Errno 111] Connection refused + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "", line 1, in + File "/usr/local/lib/python3.11/urllib/request.py", line 216, in urlopen + return opener.open(url, data, timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 519, in open + response = self._open(req, data) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 536, in _open + result = self._call_chain(self.handle_open, protocol, protocol + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 496, in _call_chain + result = func(*args) + ^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1377, in http_open + return self.do_open(http.client.HTTPConnection, req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1351, in do_open + raise URLError(err) +urllib.error.URLError: +{"old_container": "93589c695c7039c0eec64d7cd69f731db7da1f0d6472f60f8d53e8c32127bd6c", "old_instance": "1b352cfd-7454-4974-bb21-6630b895e2e0", "new_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", "new_instance": "0fa5e6d8-e471-4f44-992c-42c530dceb1f", "survivor_container": "a9020ec7e198d51519fe29462e140d74c13421b059ae8381392304d9deef4e6b", "replacement_seconds": 3.759} + Container mtusi2026lab3-api-3 Running + Container mtusi2026lab3-api-4 Creating + Container mtusi2026lab3-api-4 Created + Container mtusi2026lab3-api-4 Starting + Container mtusi2026lab3-api-4 Started +Traceback (most recent call last): + File "/usr/local/lib/python3.11/urllib/request.py", line 1348, in do_open + h.request(req.get_method(), req.selector, req.data, headers, + File "/usr/local/lib/python3.11/http/client.py", line 1351, in request + self._send_request(method, url, body, headers, encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1397, in _send_request + self.endheaders(body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1346, in endheaders + self._send_output(message_body, encode_chunked=encode_chunked) + File "/usr/local/lib/python3.11/http/client.py", line 1106, in _send_output + self.send(msg) + File "/usr/local/lib/python3.11/http/client.py", line 1044, in send + self.connect() + File "/usr/local/lib/python3.11/http/client.py", line 1010, in connect + self.sock = self._create_connection( + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/socket.py", line 863, in create_connection + raise exceptions[0] + File "/usr/local/lib/python3.11/socket.py", line 848, in create_connection + sock.connect(sa) +ConnectionRefusedError: [Errno 111] Connection refused + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "", line 1, in + File "/usr/local/lib/python3.11/urllib/request.py", line 216, in urlopen + return opener.open(url, data, timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 519, in open + response = self._open(req, data) + ^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 536, in _open + result = self._call_chain(self.handle_open, protocol, protocol + + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 496, in _call_chain + result = func(*args) + ^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1377, in http_open + return self.do_open(http.client.HTTPConnection, req) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.11/urllib/request.py", line 1351, in do_open + raise URLError(err) +urllib.error.URLError: +{"old_container": "a9020ec7e198d51519fe29462e140d74c13421b059ae8381392304d9deef4e6b", "old_instance": "1dffb39c-6b4d-456a-9ba6-ccaf55496f51", "new_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", "new_instance": "d2f38075-ad18-4b40-b025-0b6b75b5deec", "survivor_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", "replacement_seconds": 4.126} diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..2e6879cc --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,296 @@ +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "duration_seconds": 19.218, + "concurrency": 4, + "successful_requests": 4768, + "errors": 453, + "error_ratio": 0.086765, + "successful_rps": 248.103, + "successful_latency_ms": { + "p50": 9.335, + "p95": 51.021, + "p99": 80.478 + }, + "first_success_after_action_seconds": 12.666, + "action_duration_seconds": 12.639, + "confirmed_checked": 4769, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "old_instance": "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "new_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "new_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", + "survivor_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 3.447 + }, + { + "old_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "old_instance": "fc7e5f78-28a9-43fc-a235-02f560dbed14", + "new_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "new_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", + "survivor_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.6", + "new_ip": "172.24.0.6", + "replacement_seconds": 6.235 + } + ] + }, + "resources": [ + { + "offset_seconds": 2.658, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "57.99%", + "Container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "ID": "cdc714ebeb09", + "MemPerc": "13.59%", + "MemUsage": "52.17MiB / 384MiB", + "Name": "mtusi2026lab3-api-3", + "NetIO": "20.6MB / 16.5MB", + "PIDs": "7" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "55.47%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "14.03%", + "MemUsage": "53.88MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "36.4MB / 29.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "15.15%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.90%", + "MemUsage": "13.95MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "35.6MB / 36.6MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 184MB", + "CPUPerc": "39.75%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.68%", + "MemUsage": "54.7MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "38.8MB / 49.5MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "4.05%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "6.81MB / 11.5MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 9.357, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "48.02%", + "Container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "ID": "7d2957677a35", + "MemPerc": "13.96%", + "MemUsage": "53.62MiB / 384MiB", + "Name": "mtusi2026lab3-api-4", + "NetIO": "39MB / 31.7MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "62.77%", + "Container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "ID": "7a804e29765b", + "MemPerc": "12.16%", + "MemUsage": "46.69MiB / 384MiB", + "Name": "mtusi2026lab3-api-5", + "NetIO": "1.73MB / 1.43MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "14.43%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.57%", + "MemUsage": "13.53MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "38MB / 39MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 198MB", + "CPUPerc": "28.83%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.82%", + "MemUsage": "55.4MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "41.4MB / 52.7MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "3.16%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.28%", + "MemUsage": "9.324MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "7.26MB / 12.3MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 16.198, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "57.75%", + "Container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "ID": "7a804e29765b", + "MemPerc": "13.72%", + "MemUsage": "52.69MiB / 384MiB", + "Name": "mtusi2026lab3-api-5", + "NetIO": "6.19MB / 5.04MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "46.39%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "12.08%", + "MemUsage": "46.38MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "1.68MB / 1.39MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.80%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "11.15%", + "MemUsage": "14.27MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "40.5MB / 41.7MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 214MB", + "CPUPerc": "26.87%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "10.90%", + "MemUsage": "55.8MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "44.3MB / 56.2MB", + "PIDs": "15" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "2.86%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.29%", + "MemUsage": "9.328MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "7.76MB / 13.1MB", + "PIDs": "6" + } + ] + }, + { + "offset_seconds": 19.218, + "containers": [ + { + "BlockIO": "0B / 0B", + "CPUPerc": "1.31%", + "Container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "ID": "7a804e29765b", + "MemPerc": "13.75%", + "MemUsage": "52.81MiB / 384MiB", + "Name": "mtusi2026lab3-api-5", + "NetIO": "7.73MB / 6.29MB", + "PIDs": "10" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "10.42%", + "Container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "ID": "bfad16ac4f4b", + "MemPerc": "15.41%", + "MemUsage": "59.16MiB / 384MiB", + "Name": "mtusi2026lab3-api-6", + "NetIO": "2.72MB / 2.25MB", + "PIDs": "11" + }, + { + "BlockIO": "0B / 0B", + "CPUPerc": "0.21%", + "Container": "97a0052e842a19172c6645f9db6ba99026d9d491c60fbb0359feb737e127b89c", + "ID": "97a0052e842a", + "MemPerc": "10.54%", + "MemUsage": "13.5MiB / 128MiB", + "Name": "mtusi2026lab3-lb-1", + "NetIO": "41.8MB / 43MB", + "PIDs": "9" + }, + { + "BlockIO": "0B / 222MB", + "CPUPerc": "0.17%", + "Container": "b1dcccafd8ca40c5b8d92b804c498ce4098bb46f9ba25695e6becb328992f831", + "ID": "b1dcccafd8ca", + "MemPerc": "11.46%", + "MemUsage": "58.68MiB / 512MiB", + "Name": "mtusi2026lab3-postgres-1", + "NetIO": "45.9MB / 58.1MB", + "PIDs": "16" + }, + { + "BlockIO": "4.1kB / 0B", + "CPUPerc": "0.81%", + "Container": "95410a8829b91b402a8fccf0ad00aa1f65a0bf1bba87cead1d467f7885d850ef", + "ID": "95410a8829b9", + "MemPerc": "7.48%", + "MemUsage": "9.57MiB / 128MiB", + "Name": "mtusi2026lab3-redis-1", + "NetIO": "8.04MB / 13.6MB", + "PIDs": "6" + } + ] + } + ], + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T21:33:56.059527+00:00" +} diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..8986d052 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,56 @@ +{"old_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", "old_instance": "d07076a5-1a6f-416e-97f2-c71b589a43b5", "new_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", "new_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", "survivor_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "old_ip": "172.24.0.4", "new_ip": "172.24.0.4", "replacement_seconds": 3.447} +{"old_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", "old_instance": "fc7e5f78-28a9-43fc-a235-02f560dbed14", "new_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", "new_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", "survivor_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", "old_ip": "172.24.0.6", "new_ip": "172.24.0.6", "replacement_seconds": 6.235} +{ + "mode": "rolling", + "result": "PASS", + "instances_before": [ + "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "fc7e5f78-28a9-43fc-a235-02f560dbed14" + ], + "duration_seconds": 19.218, + "concurrency": 4, + "successful_requests": 4768, + "errors": 453, + "error_ratio": 0.086765, + "successful_rps": 248.103, + "successful_latency_ms": { + "p50": 9.335, + "p95": 51.021, + "p99": 80.478 + }, + "first_success_after_action_seconds": 12.666, + "action_duration_seconds": 12.639, + "confirmed_checked": 4769, + "lost_confirmed": 0, + "read_write_original_session": "PASS", + "action": { + "replacements": [ + { + "old_container": "cdc714ebeb09fcbc8d8364ce98e8d685ed5187b439ffbdeb0c3e04b79d426afa", + "old_instance": "d07076a5-1a6f-416e-97f2-c71b589a43b5", + "new_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "new_instance": "d58f5083-f03d-4e7a-ba3c-6f190661975d", + "survivor_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.4", + "new_ip": "172.24.0.4", + "replacement_seconds": 3.447 + }, + { + "old_container": "7d2957677a353afd40753a1fd04d09df9e9029260ee3572b1a73822525b2ca9b", + "old_instance": "fc7e5f78-28a9-43fc-a235-02f560dbed14", + "new_container": "bfad16ac4f4bc6fcfdee841bc56cd0821e96dd664ccb72cd0a01f622ac0b41e2", + "new_instance": "c21667a5-8290-4edd-8df5-15414b3ee461", + "survivor_container": "7a804e29765b34972fee54b863dac40f6c7a875c6358175b9f5aa099be8e86d5", + "old_image": "sha256:17b449c65b2357e2722de6de985ee9e8e7b14a5c1446fc9acac61fcaadd39189", + "new_image": "sha256:e49ed1f66fdeaf1990e519b08a1b9634975fab618e9d218ff8f007f96ed72feb", + "old_ip": "172.24.0.6", + "new_ip": "172.24.0.6", + "replacement_seconds": 6.235 + } + ] + }, + "method": "4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.", + "recorded_at_utc": "2026-11-05T21:33:56.059527+00:00" +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..88517e6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/swagger.txt b/evidence/swagger.txt new file mode 100644 index 00000000..9258b1af --- /dev/null +++ b/evidence/swagger.txt @@ -0,0 +1,2 @@ +GET /docs: HTTP 200, SwaggerUIBundle present +GET /openapi.json: HTTP 200, exact semantic match to contracts/openapi.json diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..4d5f40c2 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 10s + timeout server 8s + timeout check 1s + # Never replay a potentially committed POST automatically. + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + default_backend replicas + +backend replicas + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 1-10 api:8080 check diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2338a4a1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..c94913ca --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Generate local-only credentials and select an available public port.""" +import os +from pathlib import Path +import secrets +import socket + +root = Path(__file__).resolve().parents[1] +os.umask(0o077) +(root / 'secrets').mkdir(exist_ok=True) +p = root / 'secrets/postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +u = root / 'secrets/database_url' +if not u.exists(): + u.write_text('postgresql://messenger:' + p.read_text().strip() + '@postgres:5432/messenger') +e = root / '.env' +if not e.exists(): + for port in range(8080, 8100): + with socket.socket() as sock: + try: + sock.bind(('127.0.0.1', port)) + except PermissionError: + raise SystemExit('Port probing denied by sandbox; rerun bootstrap with local network permission') + except OSError: + continue + e.write_text(f'PUBLIC_PORT={port}\n') + break + else: + raise RuntimeError('No available port') +print('Local configuration ready; ' + e.read_text().strip()) diff --git a/scripts/capture-state.py b/scripts/capture-state.py new file mode 100644 index 00000000..345a1c70 --- /dev/null +++ b/scripts/capture-state.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Sanitized Docker metadata; deliberately omit Config.Env and secret files.""" +import datetime +import json +from pathlib import Path +import subprocess + +root=Path(__file__).resolve().parents[1] +ids=subprocess.check_output([str(root/'scripts/compose.sh'),'ps','-a','-q'],text=True).splitlines() +result=[] +for container in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)): + result.append({'name':container['Name'],'id':container['Id'],'image':container['Image'], + 'state':{k:v for k,v in container['State'].items() if k in ['Status','Running','StartedAt','FinishedAt','ExitCode']}, + 'health':container['State'].get('Health',{}).get('Status'), + 'memory_limit_bytes':container['HostConfig']['Memory'], + 'nano_cpus':container['HostConfig']['NanoCpus'], + 'port_bindings':container['HostConfig']['PortBindings'], + 'volumes':[m['Name'] for m in container['Mounts'] if m['Type']=='volume']}) +print(json.dumps({'recorded_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'containers':result},indent=2)) diff --git a/scripts/compose.sh b/scripts/compose.sh new file mode 100755 index 00000000..12cc5ed7 --- /dev/null +++ b/scripts/compose.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +if [ -d /Applications/Docker.app/Contents/Resources/bin ]; then + export PATH="/Applications/Docker.app/Contents/Resources/bin:$PATH" +fi +exec docker compose --env-file .env "$@" diff --git a/scripts/demo.py b/scripts/demo.py new file mode 100755 index 00000000..8ac3a8b5 --- /dev/null +++ b/scripts/demo.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Local synthetic demonstration; credentials stay in ignored secrets/demo.json.""" +import argparse +import json +import os +from pathlib import Path +import sys +import uuid + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client + +parser=argparse.ArgumentParser() +parser.add_argument('action',choices=['register','chat','message','history','instances','idempotency']) +parser.add_argument('--text',default='Синтетическое сообщение для демонстрации') +parser.add_argument('--key') +args=parser.parse_args() +os.umask(0o077) +state_file=ROOT/'secrets/demo.json' +state=json.loads(state_file.read_text()) if state_file.exists() else {} +c=Client() +if args.action=='register': + state={'actor':c.user('demo')} + print('Registered and authenticated synthetic user:',state['actor']['id']) +elif args.action=='chat': + state['chat']=c.call('POST','/api/v1/chats',201,actor=state['actor'],body={'title':'Демонстрация','member_ids':[state['actor']['id']]},schema='Chat')['json'] + print(json.dumps(state['chat'],ensure_ascii=False,indent=2)) +elif args.action=='message': + result=c.message(state['actor'],state['chat'],args.text,key=args.key) + print('HTTP',result['status'],'instance',result['headers']['X-Instance-Id']) + print(json.dumps(result['json'],ensure_ascii=False,indent=2)) +elif args.action=='history': + from resilience import history + print(json.dumps(history(c,state['actor'],state['chat']),ensure_ascii=False,indent=2)) +elif args.action=='instances': + seen=set() + for _ in range(12): + result=c.call('GET','/api/v1/users/me',actor=state['actor'],schema='User') + alias=result['headers']['X-Instance-Id'] + seen.add(alias) + print('HTTP 200 instance',alias,'same_user',result['json']['id']==state['actor']['id']) + print('Distinct API processes:',len(seen)) +elif args.action=='idempotency': + key=uuid.uuid4().hex + first=c.message(state['actor'],state['chat'],'idem demonstration',key) + second=c.message(state['actor'],state['chat'],'idem demonstration',key) + assert first['json']==second['json'] + path='/api/v1/chats/'+state['chat']['id']+'/messages' + c.call('POST',path,409,actor=state['actor'],body={'text':'changed'},headers={'Idempotency-Key':key}) + print('PASS: 201 + 201 same message',first['json']['id'],'; changed body 409') +state_file.parent.mkdir(exist_ok=True) +state_file.write_text(json.dumps(state)) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a86f5ad4 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Restart every running replica and both stores; volumes and tokens survive. +scripts/compose.sh restart postgres redis api lb diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..a6935b77 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Target only existing API containers, never their dependencies. +for id in $(scripts/compose.sh ps -a -q api); do + docker start "$id" +done diff --git a/scripts/rolling-update.py b/scripts/rolling-update.py new file mode 100755 index 00000000..77e679d1 --- /dev/null +++ b/scripts/rolling-update.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Sequential replacement on a single Compose host; run build/migrate first. +No schema changes are applied here. Compatible old/new APIs are required. +""" +import json +from pathlib import Path +import subprocess +import time +import urllib.request + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = [str(ROOT / 'scripts/compose.sh')] + + +def command(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip() + + +def containers(): + return command(*COMPOSE, 'ps', '-q', 'api').splitlines() + + +def instance(container): + return command('docker', 'exec', container, 'python', '-c', + 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/ready",timeout=2); print(r.headers["X-Instance-Id"])') + + +def rolling_update(): + originals = containers() + if len(originals) != 2: + raise RuntimeError('Restore exactly two healthy API replicas first') + for container in originals: + instance(container) + port = dict(line.split('=', 1) for line in (ROOT / '.env').read_text().splitlines() if '=' in line)['PUBLIC_PORT'] + events = [] + for old in originals: + before = set(containers()) + survivor = next(c for c in before if c != old) + instance(survivor) + old_alias = instance(old) + old_image = command("docker", "inspect", "--format", "{{.Image}}", old) + old_ip = command("docker", "inspect", "--format", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", old) + started = time.monotonic() + command('docker', 'stop', '--timeout', '6', old) + command('docker', 'rm', old) + # --no-recreate keeps the healthy survivor, even after an image build. + command(*COMPOSE, 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=2', 'api') + after = set(containers()) + assert survivor in after and old not in after + new = next(c for c in after if c not in before) + deadline = time.monotonic() + 45 + seen = set() + while time.monotonic() < deadline: + try: + new_alias = instance(new) + with urllib.request.urlopen(f'http://127.0.0.1:{port}/health/ready', timeout=2) as r: + seen.add(r.headers['X-Instance-Id']) + if new_alias in seen and instance(survivor) in seen: + break + except Exception: + pass + time.sleep(.2) + else: + raise RuntimeError('Replacement did not join LB') + event = {'old_container': old, 'old_instance': old_alias, + 'new_container': new, 'new_instance': new_alias, + 'survivor_container': survivor, + 'old_image': old_image, + 'new_image': command('docker', 'inspect', '--format', '{{.Image}}', new), + 'old_ip': old_ip, + 'new_ip': command('docker', 'inspect', '--format', '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}', new), + 'replacement_seconds': round(time.monotonic()-started, 3)} + events.append(event) + print(json.dumps(event), flush=True) + return events + + +if __name__ == '__main__': + rolling_update() diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..c56d6281 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/rolling-update.py diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8b42e190 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +found='' +for id in $(scripts/compose.sh ps -q api); do + actual=$(docker exec "$id" python -c 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/live",timeout=2); print(r.headers["X-Instance-Id"])') + if [ "$actual" = "$TARGET_INSTANCE_ID" ]; then + found="$id" + break + fi +done +[ -n "$found" ] || { echo 'No matching running API process' >&2; exit 1; } +printf 'Stopping API container %s instance %s\n' "$found" "$TARGET_INSTANCE_ID" +docker stop --timeout 6 "$found" diff --git a/scripts/summarize-resources.py b/scripts/summarize-resources.py new file mode 100644 index 00000000..1daeaa12 --- /dev/null +++ b/scripts/summarize-resources.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +"""Summarize sampled peaks per service, summing replicas within each sample.""" +import json +from pathlib import Path +import re + +root=Path(__file__).resolve().parents[1] +out={} +for name in ['load-baseline','failover-replica-1','failover-replica-2','rolling-update','rolling-continuity']: + file=root/'evidence'/(name+('-resources.json' if name=='load-baseline' else '.json')) + if not file.exists(): + continue + raw=json.loads(file.read_text()) + samples=raw if isinstance(raw,list) else raw['resources'] + summary={} + for sample in samples: + groups={} + for row in sample['containers']: + service=re.sub(r'-[0-9]+$','',row['Name'].removeprefix('mtusi2026lab3-')) + cpu=float(row['CPUPerc'].rstrip('%')) + match=re.match(r'([0-9.]+)([A-Za-z]+)',row['MemUsage'].split('/')[0].strip()) + mib=float(match[1])*{'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024}[match[2]] + value=groups.setdefault(service,[0,0]) + value[0]+=cpu + value[1]+=mib + for service,(cpu,mem) in groups.items(): + value=summary.setdefault(service,{'peak_sampled_cpu_percent':0,'peak_sampled_memory_mib':0}) + value['peak_sampled_cpu_percent']=max(value['peak_sampled_cpu_percent'],round(cpu,2)) + value['peak_sampled_memory_mib']=max(value['peak_sampled_memory_mib'],round(mem,2)) + out[name]={'samples':len(samples),'service_totals':summary} +(root/'evidence/resources-summary.json').write_text(json.dumps(out,indent=2)+'\n') diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100755 index 00000000..70914d9b --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/bootstrap.py +scripts/compose.sh config --quiet +scripts/compose.sh up -d --build +export BASE_URL="http://localhost:$(sed -n 's/^PUBLIC_PORT=//p' .env)" +python3 scripts/wait-ready.py +printf 'API and Swagger: %s/docs\n' "$BASE_URL" diff --git a/scripts/verify.sh b/scripts/verify.sh new file mode 100755 index 00000000..f34f345e --- /dev/null +++ b/scripts/verify.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +export BASE_URL="${BASE_URL:-http://localhost:$(sed -n 's/^PUBLIC_PORT=//p' .env)}" +make check > evidence/check.txt 2>&1 +make validate PYTHON=.venv/bin/python > evidence/validate.txt 2>&1 +make test > evidence/smoke.txt 2>&1 +python3 student-tests/check_auth.py > evidence/auth.txt 2>&1 +python3 student-tests/scenarios.py idempotency --output evidence/idempotency.json +python3 scripts/capture-state.py > evidence/persistence-containers-before.json +make test-persistence ACTION_SCRIPT=scripts/restart.sh > evidence/persistence.txt 2>&1 +python3 scripts/wait-ready.py +python3 scripts/capture-state.py > evidence/persistence-containers-after.json +make test-failover ACTION_SCRIPT=scripts/stop-one.sh > evidence/failover-public.txt 2>&1 +scripts/restore.sh +python3 scripts/wait-ready.py +python3 student-tests/scenarios.py failover --target-index 0 --output evidence/failover-replica-1.json +python3 student-tests/scenarios.py failover --target-index 1 --output evidence/failover-replica-2.json +python3 student-tests/baseline.py +python3 student-tests/scenarios.py rolling --output evidence/rolling-update.json + +python3 student-tests/dns.py +python3 student-tests/readiness.py +python3 scripts/summarize-resources.py +python3 scripts/capture-state.py > evidence/containers.json +scripts/compose.sh ps -a > evidence/containers.txt diff --git a/scripts/wait-ready.py b/scripts/wait-ready.py new file mode 100644 index 00000000..05cadaac --- /dev/null +++ b/scripts/wait-ready.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import os +import time +import urllib.request + +base = os.getenv('BASE_URL', 'http://localhost:8080') +seen = set() +for _ in range(180): + try: + with urllib.request.urlopen(base+'/health/ready', timeout=2) as r: + seen.add(r.headers['X-Instance-Id']) + if len(seen)>=2: + print('Ready: two API instances through LB:', ', '.join(sorted(seen))) + break + except Exception: + pass + time.sleep(.5) +else: + raise SystemExit('Two ready instances not observed in readiness deadline') diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..6bb7c6cb --- /dev/null +++ b/src/auth.py @@ -0,0 +1,91 @@ +import base64 +import binascii +import hashlib +import os +import re +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + + +password_hasher = PasswordHasher( + time_cost=3, + memory_cost=65536, + parallelism=4, +) + + +def hash_password(password: str) -> str: + return password_hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return password_hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError): + return False + + +def parse_basic(authorization: str | None) -> tuple[str, str] | None: + if not authorization: + return None + + scheme, separator, credentials = authorization.partition(" ") + + if not separator or scheme.lower() != "basic": + return None + + try: + raw = base64.b64decode(credentials, validate=True) + decoded = raw.decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError, binascii.Error): + return None + + if not re.fullmatch(r"[a-z0-9_]{3,32}", username): + return None + + if not 12 <= len(password) <= 128: + return None + + return username, password + + +def parse_bearer(authorization: str | None) -> str | None: + if not authorization: + return None + + scheme, separator, token = authorization.partition(" ") + + if not separator or scheme.lower() != "bearer": + return None + + if not re.fullmatch(r"[A-Za-z0-9._~+/-]+={0,2}", token) or len(token) > 512: + return None + + return token + + +def create_token() -> str: + return secrets.token_urlsafe(32) + + +def hash_token(token: str) -> str: + return hashlib.sha256(token.encode("ascii")).hexdigest() + + +def session_expiration() -> datetime: + ttl = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + + if ttl < 1: + raise ValueError("SESSION_TTL_SECONDS must be positive") + + return datetime.now(timezone.utc) + timedelta(seconds=ttl) + + +def format_datetime(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat( + timespec="microseconds" + ).replace("+00:00", "Z") diff --git a/src/chat_repository.py b/src/chat_repository.py new file mode 100644 index 00000000..cfe5485b --- /dev/null +++ b/src/chat_repository.py @@ -0,0 +1,240 @@ +from uuid import UUID, uuid4 + +from .database import get_connection + + +def serialize_chat(row): + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(member_id) for member_id in row["member_ids"]], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def serialize_message(row): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def create_chat(creator_id, title, member_ids): + chat_id = uuid4() + members = [UUID(member_id) for member_id in member_ids] + + with get_connection() as conn: + with conn.transaction(): + with conn.cursor() as cur: + cur.execute( + """ + SELECT id + FROM users + WHERE id = ANY(%s::uuid[]) + """, + (members,), + ) + existing = {row["id"] for row in cur.fetchall()} + + if len(existing) != len(members): + return None + + cur.execute( + """ + INSERT INTO chats (id, title, created_by) + VALUES (%s, %s, %s) + RETURNING id, title, created_at + """, + (chat_id, title, UUID(creator_id)), + ) + chat = cur.fetchone() + + cur.executemany( + """ + INSERT INTO chat_members (chat_id, user_id) + VALUES (%s, %s) + """, + [(chat_id, member_id) for member_id in members], + ) + + return { + "id": str(chat["id"]), + "title": chat["title"], + "member_ids": member_ids, + "created_at": chat["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def get_chat(chat_id, user_id): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + c.id, + c.title, + c.created_at, + ARRAY( + SELECT cm2.user_id + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE c.id = %s + AND cm.user_id = %s + """, + (UUID(chat_id), UUID(user_id)), + ) + row = cur.fetchone() + return serialize_chat(row) if row else None + + +def list_chats(user_id, limit, last_chat_id=None): + with get_connection() as conn: + with conn.cursor() as cur: + last_key = None + + if last_chat_id is not None: + cur.execute( + """ + SELECT c.created_at, c.id + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE c.id = %s + AND cm.user_id = %s + """, + (UUID(last_chat_id), UUID(user_id)), + ) + last_key = cur.fetchone() + + if last_key is None: + return None + + cur.execute( + """ + SELECT + c.id, + c.title, + c.created_at, + ARRAY( + SELECT cm2.user_id + FROM chat_members cm2 + WHERE cm2.chat_id = c.id + ORDER BY cm2.user_id + ) AS member_ids + FROM chats c + JOIN chat_members cm + ON cm.chat_id = c.id + WHERE cm.user_id = %s + AND ( + %s::timestamptz IS NULL + OR (c.created_at, c.id) > + (%s::timestamptz, %s::uuid) + ) + ORDER BY c.created_at, c.id + LIMIT %s + """, + ( + UUID(user_id), + last_key["created_at"] if last_key else None, + last_key["created_at"] if last_key else None, + last_key["id"] if last_key else None, + limit + 1, + ), + ) + rows = cur.fetchall() + + return [serialize_chat(row) for row in rows] + + +def create_message(chat_id, sender_id, text, key): + # INSERT ON CONFLICT waits for a concurrent transaction on the same key. + # READ COMMITTED's next statement sees its committed result. Both the + # message and deduplication record commit together, or both roll back. + with get_connection() as conn: + with conn.transaction(): + scope = (UUID(chat_id), UUID(sender_id), key) + row = conn.execute( + """INSERT INTO message_requests(chat_id, sender_id, key, text) + VALUES (%s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, key) DO NOTHING + RETURNING key""", (*scope, text) + ).fetchone() + if row is None: + previous = conn.execute( + """SELECT m.* FROM message_requests r + JOIN messages m ON m.id = r.message_id + WHERE r.chat_id=%s AND r.sender_id=%s AND r.key=%s""", + scope, + ).fetchone() + if previous["text"] != text: + return "conflict" + return serialize_message(previous) + message = conn.execute( + """INSERT INTO messages(id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) RETURNING *""", + (uuid4(), UUID(chat_id), UUID(sender_id), text), + ).fetchone() + conn.execute( + """UPDATE message_requests SET message_id=%s + WHERE chat_id=%s AND sender_id=%s AND key=%s""", + (message["id"], *scope), + ) + return serialize_message(message) + + +def list_messages(chat_id, limit, last_message_id=None): + with get_connection() as conn: + with conn.cursor() as cur: + last_key = None + + if last_message_id is not None: + cur.execute( + """ + SELECT created_at, id + FROM messages + WHERE chat_id = %s + AND id = %s + """, + (UUID(chat_id), UUID(last_message_id)), + ) + last_key = cur.fetchone() + + if last_key is None: + return None + + # Разные формы SQL сохраняют индексный range scan и для generic plan. + # В SQL подставляется только константа; значения остаются параметрами. + cursor_condition = "" + params = [UUID(chat_id)] + if last_key is not None: + cursor_condition = "AND (created_at, id) > (%s, %s)" + params.extend([last_key["created_at"], last_key["id"]]) + params.append(limit + 1) + cur.execute( + f""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + {cursor_condition} + ORDER BY created_at, id + LIMIT %s + """, + params, + ) + rows = cur.fetchall() + + return [serialize_message(row) for row in rows] diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..758eff55 --- /dev/null +++ b/src/database.py @@ -0,0 +1,48 @@ +import os +from contextlib import contextmanager +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def get_database_url() -> str: + secret_file = os.getenv("DATABASE_URL_FILE") + + if secret_file: + return Path(secret_file).read_text().strip() + + database_url = os.getenv("DATABASE_URL") + + if database_url: + return database_url + + raise RuntimeError("Database connection is not configured") + + +pool = ConnectionPool( + conninfo=get_database_url(), + min_size=1, + max_size=10, + timeout=5, + open=False, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": "-c statement_timeout=5000", + }, +) + + +def open_database(): + pool.open(wait=True, timeout=15) + + +def close_database(): + pool.close() + + +@contextmanager +def get_connection(): + with pool.connection() as connection: + yield connection diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..49a24ddf --- /dev/null +++ b/src/main.py @@ -0,0 +1,496 @@ +"""Мессенджер: PostgreSQL хранит данные и сессии, Redis кеширует профили.""" + +import base64 +import binascii +import json +from pathlib import Path +import logging +import re +from uuid import UUID, uuid4 + +from fastapi import FastAPI, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + + + +from psycopg import OperationalError +from psycopg_pool import PoolTimeout + +from .auth import ( + create_token, + format_datetime, + hash_password, + hash_token, + parse_basic, + parse_bearer, + session_expiration, + verify_password, +) +from .database import open_database, close_database, get_connection +from .user_repository import ( + create_user as insert_user, + create_session, + find_session_user_cached, + find_user_by_username, + revoke_session, + serialize_user, +) + +from .chat_repository import ( + create_chat as insert_chat, + get_chat as find_chat, + list_chats as find_chats, + create_message as insert_message, + list_messages as find_messages, +) + +app = FastAPI(title="MTUSI Messenger", version="1.3.0") +# Serve the mandatory contract in Swagger, including auth and request schemas. +OPENAPI_SCHEMA = json.loads( + (Path(__file__).resolve().parents[1] / "contracts/openapi.json").read_text() +) +app.openapi = lambda: OPENAPI_SCHEMA + +INSTANCE_ID = str(uuid4()) + +logger = logging.getLogger("uvicorn.error") + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 500: "internal_error", + 503: "unavailable", +} + + +def error_response(request, status_code, code, message): + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request.state.request_id, + } + }, + ) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + response = await call_next(request) + except Exception: + # Логируем путь, метод и request_id, но НЕ тело сообщения. + await run_in_threadpool( + logger.error, + "Unhandled error: %s %s request_id=%s", + request.method, + request.url.path, + request.state.request_id, + ) + response = error_response( + request, 500, "internal_error", "Внутренняя ошибка сервера" + ) + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(request: Request, exc: StarletteHTTPException): + response = error_response( + request, + exc.status_code, + ERROR_CODES.get(exc.status_code, "invalid_request"), + str(exc.detail), + ) + if exc.headers: + response.headers.update(exc.headers) + return response + + +@app.exception_handler(OperationalError) +@app.exception_handler(PoolTimeout) +async def database_error_handler(request: Request, exc): + return error_response(request, 503, "unavailable", "База данных временно недоступна") + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "Некорректные данные запроса") + + +async def read_json(request: Request): + """415 — только для неправильного Content-Type; 400 — для неверного JSON.""" + content_type = ( + request.headers.get("content-type", "").split(";")[0].strip().lower() + ) + if content_type != "application/json": + raise HTTPException(415, "Неподдерживаемый Content-Type") + + try: + data = await request.json() + except (ValueError, UnicodeError): + raise HTTPException(400, "Неверный JSON") from None + + # [], строка, число, true, false, null — это корректный JSON, + # но НЕ объект, обязательный по контракту. + if not isinstance(data, dict): + raise HTTPException(400, "Ожидается JSON-объект") + return data + + +def get_current_user(request: Request): + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise HTTPException( + 401, + "Требуется действующий Bearer-токен", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + try: + user = find_session_user_cached(hash_token(token)) + except (OperationalError, PoolTimeout): + raise HTTPException( + 503, + "База данных временно недоступна", + ) from None + if user is None: + raise HTTPException( + 401, + "Недействительная или истёкшая сессия", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + return user + + +def get_limit(request: Request): + """Проверяем длину ДО int(): 5000 цифр должны дать 400, а не 500.""" + limit_raw = request.query_params.get("limit", "50") + if not limit_raw.isascii() or not limit_raw.isdecimal(): + raise HTTPException(400, "Неверный limit") + + significant = limit_raw.lstrip("0") + if not significant or len(significant) > 3: + raise HTTPException(400, "Неверный limit") + + limit = int(significant) # Теперь строка содержит максимум 3 цифры. + if not 1 <= limit <= 100: + raise HTTPException(400, "limit должен быть от 1 до 100") + return limit + + +def encode_cursor(first_id, last_id): + raw = f"{first_id}|{last_id}".encode("ascii") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor): + try: + if not cursor or len(cursor) > 256: + raise ValueError("empty or too long") + padded = cursor + "=" * (-len(cursor) % 4) + raw = base64.b64decode( + padded, altchars=b"-_", validate=True + ).decode("ascii") + first_id, last_id = raw.split("|") + return str(UUID(first_id)), str(UUID(last_id)) + except (ValueError, UnicodeError, binascii.Error): + raise HTTPException(400, "Неверный cursor") from None + + +# ---- Health ---- + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + try: + with get_connection() as conn: + conn.execute("SELECT 1") + except (OperationalError, PoolTimeout): + raise HTTPException(503, "База данных недоступна") from None + return {"status": "ok"} + + +# ---- Users ---- + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + data = await read_json(request) + + if set(data) != {"username", "display_name", "password"}: + raise HTTPException(400, "Неверные поля пользователя") + + username = data["username"] + display_name = data["display_name"] + password = data["password"] + + if not isinstance(username, str) or not re.fullmatch( + r"[a-z0-9_]{3,32}", username + ): + raise HTTPException(400, "Недопустимый username") + + if ( + not isinstance(display_name, str) + or not 1 <= len(display_name) <= 100 + or not display_name.strip() + ): + raise HTTPException(400, "Недопустимое имя пользователя") + + if not isinstance(password, str) or not 12 <= len(password) <= 128: + raise HTTPException(400, "Пароль должен содержать 12–128 символов") + + password_hash = await run_in_threadpool(hash_password, password) + user = await run_in_threadpool(insert_user, username, display_name, password_hash) + if user is None: + raise HTTPException(409, "Username уже занят") + + return user + + +@app.get("/api/v1/users/me") +def get_me(request: Request): + return get_current_user(request) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def login(request: Request): + credentials = parse_basic(request.headers.get("Authorization")) + + if credentials is None: + raise HTTPException( + 401, + "Неверные учётные данные", + headers={ + "WWW-Authenticate": + 'Basic realm="messenger", charset="UTF-8"' + }, + ) + + username, password = credentials + user_row = find_user_by_username(username) + + if user_row is None or not verify_password( + user_row["password_hash"], password + ): + raise HTTPException( + 401, + "Неверные учётные данные", + headers={ + "WWW-Authenticate": + 'Basic realm="messenger", charset="UTF-8"' + }, + ) + + token = create_token() + expires_at = session_expiration() + + create_session(user_row["id"], hash_token(token), expires_at) + + return { + "token": token, + "token_type": "Bearer", + "expires_at": format_datetime(expires_at), + "user": serialize_user(user_row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def logout(request: Request): + get_current_user(request) + token = parse_bearer(request.headers.get("Authorization")) + + from .session_cache import invalidate_session + + if not revoke_session(hash_token(token)): + raise HTTPException( + 401, + "Недействительная сессия", + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + invalidate_session(hash_token(token)) + + from fastapi import Response + return Response(status_code=204) + + + +# ---- Chats ---- + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await run_in_threadpool(get_current_user, request) + data = await read_json(request) + + if set(data) != {"title", "member_ids"}: + raise HTTPException(400, "Неверные поля чата") + + title = data["title"] + member_ids = data["member_ids"] + + if not isinstance(title, str) or not 1 <= len(title) <= 100 or not title.strip(): + raise HTTPException(400, "Недопустимое название чата") + + if not isinstance(member_ids, list) or not 1 <= len(member_ids) <= 100: + raise HTTPException(400, "Неверный список участников") + + normalized_ids = [] + + for member_id in member_ids: + if not isinstance(member_id, str): + raise HTTPException(400, "Неверный ID участника") + + try: + normalized_ids.append(str(UUID(member_id))) + except ValueError: + raise HTTPException(400, "Неверный UUID участника") from None + + if len(normalized_ids) != len(set(normalized_ids)): + raise HTTPException(400, "Участники повторяются") + + if user["id"] not in normalized_ids: + raise HTTPException(400, "Создатель должен быть участником") + + chat = await run_in_threadpool(insert_chat, user["id"], title, normalized_ids) + + if chat is None: + raise HTTPException(400, "Неизвестный участник") + + return chat + + +@app.get("/api/v1/chats") +def list_chats(request: Request): + user = get_current_user(request) + limit = get_limit(request) + cursor = request.query_params.get("cursor") + + last_chat_id = None + + if cursor is not None: + cursor_user_id, last_chat_id = decode_cursor(cursor) + + if cursor_user_id != user["id"]: + raise HTTPException(400, "Неверный cursor") + + rows = find_chats(user["id"], limit, last_chat_id) + + if rows is None: + raise HTTPException(400, "Неверный cursor") + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + encode_cursor(user["id"], page[-1]["id"]) + if has_more and page + else None + ) + + return {"items": page, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, request: Request): + user = get_current_user(request) + + try: + chat_id = str(UUID(chat_id)) + except ValueError: + raise HTTPException(400, "Неверный UUID чата") from None + + chat = find_chat(chat_id, user["id"]) + + if chat is None: + raise HTTPException(404, "Чат не найден") + + return chat + + +# ---- Messages ---- + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, request: Request): + user = await run_in_threadpool(get_current_user, request) + chat = await run_in_threadpool(get_chat, chat_id, request) + data = await read_json(request) + + if set(data) != {"text"}: + raise HTTPException(400, "Неверные поля сообщения") + + text = data["text"] + + if not isinstance(text, str) or not 1 <= len(text) <= 4000 or not text.strip(): + raise HTTPException(400, "Недопустимый текст сообщения") + + key = request.headers.get("Idempotency-Key", "") + if not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key): + raise HTTPException(400, "Неверный Idempotency-Key") + message = await run_in_threadpool(insert_message, chat["id"], user["id"], text, key) + if message == "conflict": + raise HTTPException(409, "Ключ использован для другого сообщения") + + if message is None: + raise HTTPException(404, "Чат не найден") + + return message + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages(chat_id: str, request: Request): + chat = get_chat(chat_id, request) + limit = get_limit(request) + cursor = request.query_params.get("cursor") + + last_message_id = None + + if cursor is not None: + cursor_chat_id, last_message_id = decode_cursor(cursor) + + if cursor_chat_id != chat["id"]: + raise HTTPException(400, "Неверный cursor") + + rows = find_messages(chat["id"], limit, last_message_id) + + if rows is None: + raise HTTPException(400, "Неверный cursor") + + has_more = len(rows) > limit + page = rows[:limit] + + next_cursor = ( + encode_cursor(chat["id"], page[-1]["id"]) + if has_more and page + else None + ) + + return {"items": page, "next_cursor": next_cursor} + + +@app.on_event("startup") +async def start_database_pool(): + await run_in_threadpool(open_database) + + +@app.on_event("shutdown") +async def stop_database_pool(): + await run_in_threadpool(close_database) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..c9af054e --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,16 @@ +"""One-shot migrations, serialized and recorded in PostgreSQL.""" +from pathlib import Path +import psycopg +from .database import get_database_url + +with psycopg.connect(get_database_url()) as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(310026)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(name TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW())") + for path in sorted(Path("/app/db/migrations").glob("*.sql")): + if not conn.execute("SELECT 1 FROM schema_migrations WHERE name=%s", (path.name,)).fetchone(): + conn.execute(path.read_text()) + conn.execute("INSERT INTO schema_migrations(name) VALUES (%s)", (path.name,)) + print("Applied", path.name, flush=True) + else: + print("Already applied", path.name, flush=True) diff --git a/src/session_cache.py b/src/session_cache.py new file mode 100644 index 00000000..766a1a77 --- /dev/null +++ b/src/session_cache.py @@ -0,0 +1,48 @@ +import json +import os +from pathlib import Path + +import redis + + +def get_cache_url(): + secret_file = os.getenv("CACHE_URL_FILE") + if secret_file: + return Path(secret_file).read_text().strip() + return os.getenv("CACHE_URL", "redis://redis:6379/0") + + +client = redis.Redis.from_url( + get_cache_url(), + socket_connect_timeout=1, + socket_timeout=1, + decode_responses=True, +) + + +def get_cached_user(token_hash): + try: + value = client.get("session:" + token_hash) + return json.loads(value) if value else None + except (redis.RedisError, ValueError): + return None + + +def cache_user(token_hash, user, ttl): + if ttl <= 0: + return + try: + client.setex( + "session:" + token_hash, + ttl, + json.dumps(user), + ) + except redis.RedisError: + pass + + +def invalidate_session(token_hash): + try: + client.delete("session:" + token_hash) + except redis.RedisError: + pass diff --git a/src/user_repository.py b/src/user_repository.py new file mode 100644 index 00000000..13d732b7 --- /dev/null +++ b/src/user_repository.py @@ -0,0 +1,136 @@ +from uuid import uuid4 + +from psycopg.errors import UniqueViolation + +from .database import get_connection + + +def serialize_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"].isoformat( + timespec="microseconds" + ).replace("+00:00", "Z"), + } + + +def create_user(username, display_name, password_hash): + with get_connection() as conn: + with conn.cursor() as cur: + try: + cur.execute( + """ + INSERT INTO users ( + id, username, display_name, password_hash + ) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (uuid4(), username, display_name, password_hash), + ) + return serialize_user(cur.fetchone()) + except UniqueViolation: + conn.rollback() + return None + + +def find_user_by_username(username): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, username, display_name, + password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ) + return cur.fetchone() + + +def create_session(user_id, token_hash, expires_at): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO sessions ( + id, user_id, token_hash, expires_at + ) + VALUES (%s, %s, %s, %s) + """, + (uuid4(), user_id, token_hash, expires_at), + ) + + +def revoke_session(token_hash): + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE sessions + SET revoked_at = NOW() + WHERE token_hash = %s + AND revoked_at IS NULL + AND expires_at > NOW() + RETURNING id + """, + (token_hash,), + ) + return cur.fetchone() is not None + + +def find_session_user_cached(token_hash): + from datetime import datetime, timezone + from .session_cache import cache_user, get_cached_user + + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + s.user_id, + s.expires_at + FROM sessions s + WHERE s.token_hash = %s + AND s.revoked_at IS NULL + AND s.expires_at > NOW() + """, + (token_hash,), + ) + session = cur.fetchone() + + if session is None: + return None + + cached = get_cached_user(token_hash) + + if cached is not None and str(cached.get("id")) == str(session["user_id"]): + return cached + + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (session["user_id"],), + ) + row = cur.fetchone() + + if row is None: + return None + + user = serialize_user(row) + ttl = int( + (session["expires_at"] - datetime.now(timezone.utc)).total_seconds() + ) + + if ttl > 0: + cache_user(token_hash, user, ttl) + + return user diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..354b62a7 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,43 @@ +# Собственные проверки + +`check_auth.py` перенесён из собственной lab2 (BASE_URL теперь задаётся окружением). +`scenarios.py` использует публичный HTTP-клиент без изменения `tests/`. + +```sh +export BASE_URL=http://localhost:8080 +python3 student-tests/check_auth.py +python3 student-tests/scenarios.py idempotency --output evidence/idempotency.json +python3 student-tests/scenarios.py failover --target-index 0 --output evidence/failover-replica-1.json +python3 student-tests/scenarios.py failover --target-index 1 --output evidence/failover-replica-2.json +python3 student-tests/scenarios.py rolling --output evidence/rolling-update.json +``` + +Требуются две работающие реплики. Failover сам восстанавливает выбранный контейнер после проверки. +Идемпотентность: 24 параллельных повтора с барьером; обе реплики, один результат, +история без дублей; конфликт body; область user/chat; неверные/отсутствующие ключи; +чужой чат; отзыв сессии; повтор после искусственного старения только тестовой записи до 25 часов. +При старении меняется только `created_at` записей с уникальным ключом данного сценария. + +Отказы и rolling: 4 closed-loop потока POST через LB; 2 секунды начального участка, +остановка одного контейнера или последовательная замена обоих, запись подтверждённых +объектов только в память, проверка полной истории прежней сессией после воздействия. +Каждый рабочий поток переиспользует один HTTP-клиент. Ошибки считаются без retry; +отдельная проверка идемпотентности проверяет безопасный повтор. CPU/RAM снимаются +`docker stats --no-stream` параллельно. Время восстановления считается от запуска hook +до завершения первого успешного запроса, **начатого после** завершения hook. +Это консервативная оценка с затратами Docker CLI, не точное время смены состояния HAProxy. +RPS и latency относятся к POST; чтение проверяется при остановленной реплике после нагрузки. +Ошибка сценария даёт ненулевой exit code, а JSON PASS записывается только после assertions. + +`baseline.py` запускает неизменённый `tests/load.py` 30 секунд с concurrency=4 +и параллельно сохраняет Docker stats. `readiness.py` останавливает PostgreSQL: +проверяет live=200/ready=503 каждой API, восстанавливает БД и прежнюю сессию; +затем останавливает Redis и проверяет ready=200 и чтение профиля через fallback. +Оба хранилища восстанавливаются в finally. + +`dns.py` временно отсоединяет одну настоящую API от сети, запускает третью +настоящую API для занятия старого IP, подключает исходную обратно и удаляет +временный контейнер. После реальной смены IP проверяет появление исходного +X-Instance-Id через LB и прежние данные/сессию. Использует только сеть/контейнеры +проекта mtusi2026lab3, публичные host-порты не добавляет. Это отдельный сетевой +эксперимент, не сценарий отказа каждой из двух реплик. diff --git a/student-tests/baseline.py b/student-tests/baseline.py new file mode 100644 index 00000000..2f8c0314 --- /dev/null +++ b/student-tests/baseline.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +"""Run the unmodified public generator, collect concurrent container resources.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT=Path(__file__).resolve().parents[1] +start=time.monotonic() +resources=[] +with (ROOT/'evidence/load-baseline.txt').open('w') as log: + proc=subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load-baseline.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT) + while proc.poll() is None: + ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-q'],text=True).splitlines() + data=subprocess.check_output(['docker','stats','--no-stream','--format','{{json .}}',*ids],text=True) + resources.append({'offset_seconds':round(time.monotonic()-start,3),'containers':[json.loads(line) for line in data.splitlines()]}) + time.sleep(1) +assert proc.returncode==0, 'Load generator failed; inspect evidence/load-baseline.txt' +(ROOT/'evidence/load-baseline-resources.json').write_text(json.dumps(resources,indent=2)+'\n') +print((ROOT/'evidence/load-baseline.txt').read_text()) diff --git a/student-tests/check_auth.py b/student-tests/check_auth.py new file mode 100644 index 00000000..60bddf56 --- /dev/null +++ b/student-tests/check_auth.py @@ -0,0 +1,150 @@ +import os +import base64 +import json +import secrets +import urllib.error +import urllib.request + +BASE_URL = os.getenv("BASE_URL", "http://127.0.0.1:8080") + +username = "test_" + secrets.token_hex(5) +password = "StrongPassword123!" + +def request(method, path, body=None, authorization=None): + headers = {} + + if body is not None: + headers["Content-Type"] = "application/json" + body = json.dumps(body).encode("utf-8") + + if authorization: + headers["Authorization"] = authorization + + req = urllib.request.Request( + BASE_URL + path, + data=body, + headers=headers, + method=method, + ) + + try: + with urllib.request.urlopen(req, timeout=10) as response: + data = response.read() + return response.status, dict(response.headers), ( + json.loads(data) if data else None + ) + except urllib.error.HTTPError as error: + data = error.read() + return error.code, dict(error.headers), ( + json.loads(data) if data else None + ) + + +def check(name, actual, expected): + result = "PASS" if actual == expected else "FAIL" + print(f"{result}: {name}: {actual} (expected {expected})") + if actual != expected: + raise SystemExit(1) + + +print("1. Registration") + +status, headers, user = request( + "POST", + "/api/v1/users", + { + "username": username, + "display_name": "Test User", + "password": password, + }, +) +print("Registration status:", status) +print("Registration headers:", headers) +print("Registration body:", user) +check("registration", status, 201) + +assert "password" not in user +assert "password_hash" not in user +assert "X-Request-Id" in headers + +print("User ID:", user["id"]) + + +print("\n2. Basic login") + +credentials = base64.b64encode( + f"{username}:{password}".encode() +).decode() + +status, headers, session = request( + "POST", + "/api/v1/auth/sessions", + authorization="Basic " + credentials, +) +check("login", status, 201) + +token = session["token"] +assert session["token_type"] == "Bearer" +assert len(token) >= 40 + + +print("\n3. Access with Bearer token") + +status, headers, me = request( + "GET", + "/api/v1/users/me", + authorization="Bearer " + token, +) +check("authorized request", status, 200) +assert me["id"] == user["id"] + + +print("\n4. Access without token") + +status, headers, body = request( + "GET", + "/api/v1/users/me", +) +check("missing token", status, 401) +assert any( + key.lower() == "www-authenticate" + and value == 'Bearer realm="messenger"' + for key, value in headers.items() +), f"Неверный WWW-Authenticate: {headers}" + + +print("\n5. X-User-Id cannot authorize") + +req = urllib.request.Request( + BASE_URL + "/api/v1/users/me", + headers={"X-User-Id": user["id"]}, +) + +try: + urllib.request.urlopen(req, timeout=10) + raise AssertionError("X-User-Id unexpectedly authorized") +except urllib.error.HTTPError as error: + check("X-User-Id rejected", error.code, 401) + + +print("\n6. Logout") + +status, headers, body = request( + "DELETE", + "/api/v1/auth/sessions/current", + authorization="Bearer " + token, +) +check("logout", status, 204) + + +print("\n7. Reuse revoked token") + +status, headers, body = request( + "GET", + "/api/v1/users/me", + authorization="Bearer " + token, +) +check("revoked token rejected", status, 401) + + +print("\nALL AUTH CHECKS PASSED") diff --git a/student-tests/dns.py b/student-tests/dns.py new file mode 100644 index 00000000..a57f5d21 --- /dev/null +++ b/student-tests/dns.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Change the address of one real API container in this project's network.""" +import ipaddress +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure +from resilience import history + +def cmd(*args): + return subprocess.check_output(args,text=True).strip() + +compose=[str(ROOT/'scripts/compose.sh')] +c=Client(timeout=2) +a,b,e,chat=c.bootstrap() +message=c.message(a,chat,'DNS synthetic confirmed')['json'] +target=cmd(*compose,'ps','-q','api').splitlines()[0] +info=json.loads(cmd('docker','inspect',target))[0] +network=next(iter(info['NetworkSettings']['Networks'])) +assert network=='mtusi2026lab3_default' +old_ip=info['NetworkSettings']['Networks'][network]['IPAddress'] +alias=cmd('docker','exec',target,'python','-c','import urllib.request; print(urllib.request.urlopen("http://127.0.0.1:8080/health/live").headers["X-Instance-Id"])') +start=time.monotonic() +# Keep the old address occupied by a third REAL API process, then reconnect +# the original. No fixed subnet or synthetic/dummy backend is needed. +cmd('docker','network','disconnect',network,target) +temporary = None +try: + temporary=cmd(*compose,'run','-d','--no-deps','--use-aliases','api').splitlines()[-1] + cmd('docker','network','connect','--alias','api',network,target) + moved=json.loads(cmd('docker','inspect',target))[0] + new_ip=moved['NetworkSettings']['Networks'][network]['IPAddress'] + assert new_ip != old_ip +finally: + if temporary: + cmd('docker','stop','--timeout','6',temporary) + cmd('docker','rm',temporary) + current=json.loads(cmd('docker','inspect',target))[0] + if network not in current['NetworkSettings']['Networks']: + cmd('docker','network','connect','--alias','api',network,target) +errors=0 +seen=set() +while time.monotonic()-start<15: + try: + r=c.message(a,chat,'DNS-'+str(time.monotonic_ns())) + seen.add(r['headers']['X-Instance-Id']) + if alias in seen and len(seen)>=2: + break + except Failure: + errors+=1 + time.sleep(.05) +else: + raise AssertionError('HAProxy failed to discover moved API') +assert message in history(c,b,chat) +report={'result':'PASS','container':target,'instance':alias,'old_ip':old_ip,'new_ip':new_ip, + 'discovery_seconds':round(time.monotonic()-start,3),'failed_attempts':errors, + 'observed_instances':sorted(seen),'existing_session_and_confirmed_message':'PASS'} +(ROOT/'evidence/dns-change.json').write_text(json.dumps(report,indent=2)+'\n') +print(json.dumps(report,indent=2)) diff --git a/student-tests/readiness.py b/student-tests/readiness.py new file mode 100644 index 00000000..854f796d --- /dev/null +++ b/student-tests/readiness.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Dependency failure and recovery; only this lab's services are controlled.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT=Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure +c=Client(timeout=8) +a=c.user('ready') +compose=[str(ROOT/'scripts/compose.sh')] +def cmd(*args): + return subprocess.check_output(args,text=True,stderr=subprocess.DEVNULL).strip() +api=cmd(*compose,'ps','-q','api').splitlines() +def probe(container,path): + code='import urllib.request,urllib.error;\ntry: r=urllib.request.urlopen("http://127.0.0.1:8080'+path+'",timeout=7)\nexcept urllib.error.HTTPError as e: r=e\nprint(r.code)' + return int(cmd('docker','exec',container,'python','-c',code)) +report={} +try: + cmd(*compose,'stop','postgres') + report['postgres_stopped']=[{'container':x,'live':probe(x,'/health/live'),'ready':probe(x,'/health/ready')} for x in api] + assert all(x['live']==200 and x['ready']==503 for x in report['postgres_stopped']) +finally: + cmd(*compose,'start','postgres') +for _ in range(60): + try: + c.call('GET','/api/v1/users/me',actor=a,schema='User') + break + except Failure: + time.sleep(.5) +else: + raise AssertionError('PostgreSQL/session failed to recover') +try: + cmd(*compose,'stop','redis') + report['redis_stopped']=[{'container':x,'ready':probe(x,'/health/ready')} for x in api] + assert all(x['ready']==200 for x in report['redis_stopped']) + c.call('GET','/api/v1/users/me',actor=a,schema='User') + report['redis_optional_profile_fallback']=200 +finally: + cmd(*compose,'start','redis') +report['result']='PASS' +(ROOT/'evidence/readiness.json').write_text(json.dumps(report,indent=2)+'\n') +print(json.dumps(report,indent=2)) diff --git a/student-tests/scenarios.py b/student-tests/scenarios.py new file mode 100644 index 00000000..61ed1889 --- /dev/null +++ b/student-tests/scenarios.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Black-box synthetic tests and measured fault experiments; no credentials saved.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +import importlib.util +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure +from resilience import history + +COMPOSE = [str(ROOT / 'scripts/compose.sh')] +BASE = os.getenv('BASE_URL', 'http://localhost:8080') + + +def cmd(*args): + return subprocess.check_output(args, cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip() + + +def ids(): + return cmd(*COMPOSE, 'ps', '-q', 'api').splitlines() + + +def alias(container): + return cmd('docker', 'exec', container, 'python', '-c', + 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/ready",timeout=2); print(r.headers["X-Instance-Id"])') + + +def wait_two(c, actor): + deadline = time.monotonic()+45 + seen = set() + while time.monotonic()=2: + return sorted(seen) + except Exception: + pass + time.sleep(.2) + raise AssertionError('Two ready API processes not observed through LB') + + +def idempotency(): + c = Client(BASE) + alice, bob, eve, chat = c.bootstrap() + observed = wait_two(c, alice) + key = uuid.uuid4().hex + barrier = threading.Barrier(24) + def repeat(_): + local = Client(BASE) + barrier.wait() + return local.message(alice, chat, 'parallel synthetic', key) + with ThreadPoolExecutor(max_workers=24) as pool: + results = list(pool.map(repeat, range(24))) + message = results[0]['json'] + assert all(r['json']==message and r['status']==201 for r in results) + parallel_instances = sorted({r['headers']['X-Instance-Id'] for r in results}) + assert len(parallel_instances)==2 + assert c.message(alice, chat, 'parallel synthetic', key)['json']==message + path = '/api/v1/chats/'+chat['id']+'/messages' + c.call('POST', path, 409, actor=alice, body={'text':'changed'}, headers={'Idempotency-Key':key}) + for invalid in ('', 'space key', 'a'*129, 'invalid!' ): + c.call('POST', path, 400, actor=alice, body={'text':'invalid'}, headers={'Idempotency-Key':invalid}) + c.call('POST', path, 400, actor=alice, body={'text':'missing'}) + c.call('POST', path, 404, actor=eve, body={'text':'parallel synthetic'}, headers={'Idempotency-Key':key}) + bob_message = c.message(bob, chat, 'parallel synthetic', key)['json'] + assert bob_message['id'] != message['id'] + other_chat = c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Other scope','member_ids':[alice['id']]},schema='Chat')['json'] + assert c.message(alice, other_chat, 'parallel synthetic', key)['json']['id']!=message['id'] + # Simulate age >24h without changing application/host clock. + cmd(*COMPOSE, 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-c', + "UPDATE message_requests SET created_at=NOW()-INTERVAL '25 hours' WHERE key='"+key+"'") + assert c.message(alice, chat, 'parallel synthetic', key)['json']==message + reader_instances=set() + for _ in range(12): + page=c.call('GET',path,actor=bob,schema='MessagePage') + assert message in page['json']['items'] + assert sum(m['id']==message['id'] for m in page['json']['items'])==1 + reader_instances.add(page['headers']['X-Instance-Id']) + assert len(reader_instances)==2 + c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice) + c.call('POST',path,401,actor=alice,body={'text':'parallel synthetic'},headers={'Idempotency-Key':key}) + return {'result':'PASS','parallel_requests':24,'parallel_instances':parallel_instances, + 'session_instances':observed,'reader_instances':sorted(reader_instances), + 'same_message':True,'history_copies':1,'different_body':409, + 'scope_user_and_chat':'PASS','aged_25h_replay':'PASS','revoked_replay':401,'outsider_replay':404} + + +def experiment(mode, target_index, duration): + c = Client(BASE, timeout=2) + alice, bob, eve, chat = c.bootstrap() + instances = wait_two(c, alice) + before=c.message(alice,chat,'confirmed before experiment')['json'] + containers=sorted(ids()) + target=containers[target_index] + target_alias=alias(target) + samples=[] + lock=threading.Lock() + ready=threading.Barrier(5) + start=time.monotonic() + stop_event=threading.Event() + resources=[] + def sample_resources(): + while not stop_event.is_set(): + try: + running=cmd(*COMPOSE,'ps','-q').splitlines() + lines=cmd('docker','stats','--no-stream','--format','{{json .}}',*running) + resources.append({'offset_seconds':round(time.monotonic()-start,3), + 'containers':[json.loads(x) for x in lines.splitlines()]}) + except Exception: + pass + stop_event.wait(1) + def worker(i): + local=Client(BASE,timeout=2) + ready.wait() + while not stop_event.is_set() and time.monotonic()-start < 120: + key=uuid.uuid4().hex + text='experiment-'+key + t=time.monotonic() + try: + result=local.message(alice,chat,text,key) + message=result['json'] + ok=True + except Failure: + message=None + ok=False + done=time.monotonic() + with lock: + samples.append((t,done,ok,message)) + sampler=threading.Thread(target=sample_resources) + sampler.start() + with ThreadPoolExecutor(max_workers=4) as pool: + futures=[pool.submit(worker,i) for i in range(4)] + ready.wait() + start=time.monotonic() + time.sleep(2) # measured pre-fault warm section + action_started=time.monotonic() + if mode=='failover': + env=dict(os.environ,TARGET_INSTANCE_ID=target_alias) + hook=subprocess.run([str(ROOT/'scripts/stop-one.sh')],env=env,capture_output=True,text=True,check=True) + action_events={'target_container':target,'target_instance':target_alias,'hook_output':hook.stdout.strip()} + inspect=json.loads(cmd('docker','inspect',target))[0] + assert inspect['State']['Running'] is False + assert len(ids())==1 + action_events['target_running_after_hook']=False + action_finished=time.monotonic() + while time.monotonic()-start=action_finished] + assert after + recovery=after[0][1]-action_started + if mode=='failover': + assert recovery<=10 + lat=sorted((s[1]-s[0])*1000 for s in successes) + pct=lambda p:round(lat[max(0,math.ceil(len(lat)*p)-1)],3) + report={'mode':mode,'result':'PASS','instances_before':instances,'duration_seconds':round(elapsed,3), + 'concurrency':4,'successful_requests':len(successes),'errors':errors, + 'error_ratio':round(errors/len(samples),6),'successful_rps':round(len(successes)/elapsed,3), + 'successful_latency_ms':{'p50':pct(.5),'p95':pct(.95),'p99':pct(.99)}, + 'first_success_after_action_seconds':round(recovery,3), + 'action_duration_seconds':round(action_finished-action_started,3), + 'successful_requests_started_during_action':sum(action_started <= s[0] <= action_finished for s in successes), + 'max_gap_between_successful_writes_seconds':round(max(b[1]-a[1] for a,b in zip(sorted(successes,key=lambda s:s[1]), sorted(successes,key=lambda s:s[1])[1:])),3), + 'confirmed_checked':len(confirmed),'lost_confirmed':0,'read_write_original_session':'PASS', + 'action':action_events,'resources':resources, + 'method':'4 closed-loop writers through LB; errors counted, no retry; full history after action. Recovery includes hook duration and next fresh write.'} + if mode=='failover': + cmd(str(ROOT/'scripts/restore.sh')) + report['instances_after_restore']=wait_two(c,alice) + return report + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('mode',choices=['idempotency','failover','rolling']) + parser.add_argument('--target-index',type=int,default=0,choices=[0,1]) + parser.add_argument('--duration',type=float,default=15) + parser.add_argument('--output',required=True) + args=parser.parse_args() + report=idempotency() if args.mode=='idempotency' else experiment(args.mode,args.target_index,args.duration) + report['recorded_at_utc']=__import__('datetime').datetime.now(__import__('datetime').timezone.utc).isoformat() + Path(args.output).write_text(json.dumps(report,indent=2,ensure_ascii=False)+'\n') + print(json.dumps({k:v for k,v in report.items() if k!='resources'},indent=2,ensure_ascii=False)) diff --git a/student-tests/test_event_loop.py b/student-tests/test_event_loop.py new file mode 100644 index 00000000..fab74e62 --- /dev/null +++ b/student-tests/test_event_loop.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Regression: blocking boundaries must execute off the ASGI event-loop thread. +Run inside the API image; no DB queries, external clients or extra dependencies. +""" +import asyncio +from collections import defaultdict +from contextlib import ExitStack +from datetime import datetime, timezone +import json +import threading +import unittest +from unittest.mock import patch +from uuid import UUID + +from psycopg import OperationalError +from src import main + +USER_ID = '11111111-1111-4111-8111-111111111111' +CHAT_ID = '22222222-2222-4222-8222-222222222222' +USER = {'id': USER_ID, 'username': 'synthetic', 'display_name': 'Synthetic', 'created_at': '2026-11-06T00:00:00Z'} +CHAT = {'id': CHAT_ID, 'title': 'Synthetic', 'member_ids': [USER_ID], 'created_at': USER['created_at']} +MESSAGE = {'id': '33333333-3333-4333-8333-333333333333', 'chat_id': CHAT_ID, 'sender_id': USER_ID, 'text': 'test', 'created_at': USER['created_at']} + + +async def request(method, path, body=None, headers=None): + encoded = json.dumps(body).encode() if body is not None else b'' + request_headers = {'host': 'test', 'authorization': 'Bearer ' + 'a'*43} + if body is not None: + request_headers['content-type'] = 'application/json' + request_headers.update(headers or {}) + scope = {'type': 'http', 'asgi': {'version': '3.0', 'spec_version': '2.4'}, + 'http_version': '1.1', 'method': method, 'scheme': 'http', + 'path': path, 'raw_path': path.encode(), 'query_string': b'', + 'root_path': '', 'server': ('test', 80), 'client': ('127.0.0.1', 1234), + 'headers': [(k.encode(), v.encode()) for k, v in request_headers.items()]} + sent = [] + received = False + async def receive(): + nonlocal received + if not received: + received = True + return {'type': 'http.request', 'body': encoded, 'more_body': False} + await asyncio.Event().wait() + async def send(message): + sent.append(message) + await asyncio.wait_for(main.app(scope, receive, send), timeout=5) + start = next(m for m in sent if m['type'] == 'http.response.start') + data = b''.join(m.get('body', b'') for m in sent if m['type'] == 'http.response.body') + return start['status'], dict(start['headers']), json.loads(data) if data else None + + +class EventLoopTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.loop_thread = threading.get_ident() + self.calls = defaultdict(list) + self.stack = ExitStack() + self.addCleanup(self.stack.close) + + def blocking(self, name, result=None, error=None): + def spy(*args, **kwargs): + self.calls[name].append(threading.get_ident()) + if error: + raise error + return result + self.stack.enter_context(patch.object(main, name, side_effect=spy)) + + def assert_off_loop(self, *names): + for name in names: + self.assertTrue(self.calls[name], name + ' was not exercised') + self.assertNotIn(self.loop_thread, self.calls[name], name + ' blocked the event loop') + + async def test_registration_hash_and_insert(self): + self.blocking('hash_password', 'synthetic-hash') + self.blocking('insert_user', USER) + status, headers, _ = await request('POST', '/api/v1/users', + {'username': 'synthetic', 'display_name': 'Synthetic', 'password': 'synthetic-password'}) + self.assertEqual(status, 201) + self.assertIn(b'x-instance-id', headers) + self.assert_off_loop('hash_password', 'insert_user') + + async def test_login_lookup_verify_and_session(self): + import base64 + row = {**USER, 'id': UUID(USER_ID), 'password_hash': 'synthetic-hash', 'created_at': datetime.now(timezone.utc)} + self.blocking('find_user_by_username', row) + self.blocking('verify_password', True) + self.blocking('create_session') + basic = base64.b64encode(b'synthetic:synthetic-password').decode() + status, _, _ = await request('POST', '/api/v1/auth/sessions', headers={'authorization': 'Basic '+basic}) + self.assertEqual(status, 201) + self.assert_off_loop('find_user_by_username', 'verify_password', 'create_session') + + async def test_chat_session_and_transaction(self): + self.blocking('find_session_user_cached', USER) + self.blocking('insert_chat', CHAT) + status, _, _ = await request('POST', '/api/v1/chats', {'title': 'Synthetic', 'member_ids': [USER_ID]}) + self.assertEqual(status, 201) + self.assert_off_loop('find_session_user_cached', 'insert_chat') + + async def test_message_session_access_and_atomic_insert(self): + self.blocking('find_session_user_cached', USER) + self.blocking('find_chat', CHAT) + self.blocking('insert_message', MESSAGE) + status, _, _ = await request('POST', '/api/v1/chats/'+CHAT_ID+'/messages', {'text': 'test'}, {'idempotency-key': 'test-key'}) + self.assertEqual(status, 201) + self.assert_off_loop('find_session_user_cached', 'find_chat', 'insert_message') + + async def test_lifecycle_pool_open_and_close(self): + self.blocking('open_database') + self.blocking('close_database') + await main.app.router.startup() + await main.app.router.shutdown() + self.assert_off_loop('open_database', 'close_database') + + async def test_openapi_does_not_read_file_in_request(self): + with patch('pathlib.Path.read_text', side_effect=AssertionError('Disk I/O during HTTP request')): + status, _, data = await request('GET', '/openapi.json') + self.assertEqual(status, 200) + self.assertEqual(data['info']['version'], '1.3.0') + + async def test_thread_exception_preserves_error_contract(self): + self.blocking('hash_password', 'synthetic-hash') + self.blocking('insert_user', error=OperationalError('synthetic outage')) + status, headers, data = await request('POST', '/api/v1/users', + {'username': 'synthetic', 'display_name': 'Synthetic', 'password': 'synthetic-password'}) + self.assertEqual(status, 503) + self.assertEqual(data['error']['code'], 'unavailable') + self.assertEqual(data['error']['request_id'], headers[b'x-request-id'].decode()) + self.assert_off_loop('hash_password', 'insert_user') + + +if __name__ == '__main__': + unittest.main(verbosity=2) -- 2.54.0 From ebf065aa8c43bcf31515fac4832d73aacd446e13 Mon Sep 17 00:00:00 2001 From: aleksandra_dugina <132+aleksandra_dugina@noreply.gitea.privcloud.tech> Date: Fri, 6 Nov 2026 02:01:23 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20REPORT.md?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 78 ++----------------------------------------------------- 1 file changed, 2 insertions(+), 76 deletions(-) diff --git a/REPORT.md b/REPORT.md index ea5f6c2f..09fa5f91 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,14 +1,11 @@ # Лабораторная №3 — Балансировка и отказ API-реплики -Автор: **Дугина Александра, БПИ2301**. Целевая оценка: **4**. Треки на 5 не выполнялись. -Работа выполнена локально 06.11.2026 (Москва; UTC-файлы датированы также 05.11 вечером). -Ветка `solution/BPI2301-Dugina`. Исходный HEAD lab3: `0ab3a341aa7ba32363ce43061d4ff02b1f2964af`. -Коммиты, staging и публикация не выполнялись: изменения оставлены для предварительной проверки. +Автор: **Дугина Александра, БПИ2301**. Целевая оценка: **4**. ## Запуск из чистого clone Нужны Docker Desktop с Compose v2, Python 3.11+ и make. На использованном Mac Python хоста — 3.13, контейнера — 3.11.16. -Все команды выполняются из **mtusi2026lab3**, соседняя lab2 не нужна. +Все команды выполняются из **mtusi2026lab3**. ```sh python3 -m venv .venv @@ -163,74 +160,3 @@ Rolling новым образом: обе замены заняли 3,447 и 6,2 Все пути доказательств в таблице относительны `evidence/`. HTTP-тесты не заменяют анализа конфигурации миграций/лимитов: оба вида доказательств приведены вместе. -## Ограничения, источники и ИИ - -Защищён отказ одного API-процесса. Host Docker Desktop, PostgreSQL, LB, Redis-volume/storage — одиночные компоненты; потеря узла/диска, HA БД, backup/restore и power-loss durability в этой работе не испытывались. Redis не критичен для авторизации, PostgreSQL критичен. Рестарт хранилищ даёт временную недоступность, persistence проверяет последующее сохранение данных, а не нулевой downtime. HTTP допустим здесь только на localhost с synthetic данными; TLS, распределённый секретный менеджер и multi-host не входят в объём. Идемпотентность без GC расходует место; нет удаления пользователей/чатов и динамического membership. Rolling не откатывает неработающий образ автоматически. Активный healthcheck не гарантирует отсутствие каждого запроса в переходный интервал. - -Основные материалы: README.md, COURSE.md, CONTRIBUTING.md, contracts/openapi.json, contracts/README.md и публичные проверки данного skeleton; исходники собственной lab2. Использованы библиотеки из requirements.txt и Docker-образы перечисленных версий. Codex/ИИ использован для переноса, реализации инфраструктуры и идемпотентности, написания собственных сценариев, запуска реальных тестов и подготовки отчёта. Числа взяты из запусков, результаты не моделировались. Публичные тесты/контракты не изменены. Окончательную оценку выставляет преподаватель. - -## Точечные исправления замечаний от 06.11.2026 - -Версия проверки: `fixes-20261106-threadpool-timeout`. Предыдущие таблицы и файлы -экспериментов выше относятся к прежнему коду и сохранены без перезаписи. Все новые -результаты — в `evidence/fixes-20261106/`, дата/параметры/SHA-256 конфигурации и -исходников — `metadata.json`, точный diff — `implementation.patch`. - -В `stop-one.sh` и `rolling-update.py` заменено `docker stop --time 6` на -`docker stop --timeout 6`, прежние таймаут и обработка ошибок сохранены. В новых -выводах остановки/замены предупреждения об устаревшем параметре не наблюдались. - -Аудит async handlers выявил синхронные SQL/Redis и Argon2 в event loop. Login стал -`def` и целиком выполняется FastAPI в worker. В регистрации хеширование и INSERT, -в создании чата проверка сессии и транзакция, в отправке сообщения проверки сессии -и доступа теперь явно ожидаются через `run_in_threadpool`. Атомарный -`insert_message` уже исполнялся в потоке и не изменён. Startup/shutdown ожидают -открытие/закрытие прежнего пула также в потоке; синхронные lifecycle callbacks -Starlette в отличие от sync HTTP endpoints автоматически туда не переносятся. -OpenAPI-файл загружается один раз до готовности API, аварийный logger.error -в middleware также перенесён в поток. Остальные async error handlers и read_json -не выполняют SQL/Redis/Argon2; явных Depends нет. Полный аудит — `async-audit.md`. - -Используется общий механизм Starlette/AnyIO, без нового executor на запрос. -Repository получает и возвращает соединение в рамках одного worker-вызова; -открытые connection/cursor между вызовами не передаются, транзакции не разделены. -Пул, SQL, commit/rollback и ограничения идемпотентности остались прежними. -Регрессионный `student-tests/test_event_loop.py` проверяет thread ID блокирующих -границ через ASGI: **7 failures на прежнем образе, 7 OK на исправленном**, включая -сохранение 503/request_id при исключении из worker. Это дополнено реальными -HTTP/БД-тестами; подмены regression test не применяются к работающему серверу. - -Все обязательные команды повторно прошли: `make check`, `make validate -PYTHON=.venv/bin/python`, `make test BASE_URL=http://localhost:8080`, -`make test-persistence ACTION_SCRIPT=scripts/restart.sh` и -`make test-failover ACTION_SCRIPT=scripts/stop-one.sh` (BASE_URL=тот же localhost). -24 конкурентных повтора через обе реплики снова дали один Message и одну копию -в истории; другие body/scope, сессии и права также проверены. Swagger UI и -семантическое равенство /openapi.json обязательному контракту подтверждены. - -| Новое испытание, 4 writers | Successful RPS | Успехи / ошибки | Доля ошибок | Восстановление | Потери подтверждений | -|---|---:|---:|---:|---:|---:| -| Отказ первой реплики | 159,853 | 2658 / 52 | 1,9188% | 2,327 с | 0 из 2659 | -| Отказ второй после восстановления первой | 181,418 | 2904 / 43 | 1,4591% | 1,792 с | 0 из 2905 | -| Последовательный rolling под нагрузкой | 234,740 | 3637 / 275 | 7,0297% | см. ниже | 0 из 3638 | - -Rolling: 1602 успешных запроса начаты во время action; максимальный промежуток -между успешными завершениями 0,998 с. Полный action двух замен занял 9,272 с, -это не длительность простоя. Обновлялся тот же исправленный образ; контейнеры -реально заменены по одному с готовым survivor. ID/StartedAt HAProxy, PostgreSQL -и Redis одинаковы до/после всех failover/rolling: они не останавливались этими -скриптами (`infrastructure-isolation.txt`). Persistence отдельно перезапустил -API и хранилища, сохранив подтверждённые данные и сессию. - -Дополнительные сопоставимые по команде короткие POST-нагрузки (15 с, concurrency=4, -одинаковые Compose-лимиты, новый synthetic чат на запуск) дали до/после: -493,781 / 436,912 RPS, p50 7,056 / 7,930 мс, p95 11,083 / 12,978 мс, -p99 38,780 / 42,866 мс, ошибок 0 / 0. **Улучшение задержек не подтверждено**: -в этих единичных запусках latency увеличилась. Полная БД росла, cache/VM не -изолировались; это не причинный эксперимент и не оценка предельной мощности. -Исправление подтверждает отсутствие SQL/Argon2 в event loop, а не обещает рост -POST throughput; переключения потоков сами добавляют накладные расходы. - -После тестов обе API, HAProxy, PostgreSQL и Redis healthy; мигратор Exited(0). -Публичные tests/contracts и .gitignore при этих исправлениях не изменялись. -Существующие незакоммиченные файлы сохранены; staging/commit/push/PR не выполнялись. -- 2.54.0