diff --git a/.env.example b/.env.example index d32b9be0..7cecca5c 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,7 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +# Per-replica limits; adjust for the fixed-budget track 1 experiment. +API_CPU_LIMIT=1.0 +API_MEMORY_LIMIT=512M diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..2c47301a --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,34 @@ +name: Application checks +on: [push, pull_request] +permissions: + contents: read +jobs: + compose-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install -r requirements-dev.txt + - name: Create local test credentials + run: | + cp .env.example .env + ./scripts/bootstrap-secrets.sh + - name: Validate and start the application + run: | + make check validate PYTHON=python + docker compose --env-file .env config --quiet + docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180 + - name: Run API and concurrency tests + run: | + make test + make test-idempotency + - name: Verify persistence and failover + run: | + make test-persistence ACTION_SCRIPT=scripts/restart.sh + make test-failover ACTION_SCRIPT=scripts/stop-one.sh + - name: Stop test stack + if: always() + run: docker compose --env-file .env down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..83823c86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1 + +CMD ["python3", "src/server.py"] diff --git a/Makefile b/Makefile index 5d86d171..505a6c11 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://127.0.0.1:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-idempotency track1 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -28,3 +28,9 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency: + $(PYTHON) student-tests/test_idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +track1: + $(PYTHON) student-tests/track1_benchmark.py diff --git a/README.md b/README.md index 10381de4..29627f76 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Запустить несколько взаимозаменяемых экземпляров API за единой точкой входа и сохранить корректность при остановке одного из них. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +В репозитории реализован стенд лабораторной: контракт и публичные проверки дополнены приложением, Docker-инфраструктурой, сценариями отказов и исследовательским экспериментом. Критерии задания и выбранные решения описаны ниже и в [отчёте](REPORT.md). [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -12,6 +12,34 @@ **Архитектура:** Клиент → LB → API × 2+ → общие PostgreSQL и хранилище сессий. Compose или Swarm — на выбор. +## Реализация этого решения + +Используется Docker Compose: HAProxy с активной проверкой `GET /health/ready` и DNS-resolver Docker, два и более одинаковых API-контейнера, одна PostgreSQL. Публичный порт публикуется только у HAProxy на loopback. Сессии и идемпотентность находятся в PostgreSQL; локального состояния сессий у реплик нет. Миграции выполняет отдельный одноразовый сервис до старта API. + +```sh +cp .env.example .env +./scripts/bootstrap-secrets.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120 +make check +make test +make test-idempotency +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# Полная матрица трека 1: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте. +make track1 +``` + +Локальные секреты хранятся в игнорируемой папке `secrets/`; скрипт их не выводит. API использует имя контейнера как стабильный на время жизни процесса `X-Instance-Id`. Для намеренного удаления учебной БД после проверки выполните `docker compose --env-file .env down -v`; эта команда необратимо удаляет volume и не используется сценариями restart/failover. + +Перед завершением работы восстановите остановленную реплику командой `docker compose --env-file .env up -d --scale api=2 --wait`. Подробные методика, ограничения и доказательства указаны в `REPORT.md`. + +Для проверки поочерёдной замены экземпляров под нагрузкой запустите в одном терминале `python3 tests/load.py --duration 30 --concurrency 8 --output evidence/rolling-restart-load.json`, а пока она работает — в другом `scripts/rolling-restart.sh`. Скрипт перезапускает по одному контейнеру, не останавливая вторую реплику, и ждёт readiness обоих экземпляров. + +Перед `make track1` запустите Compose-стенд и не допускайте параллельных записей в БД: скрипт останавливает API, создаёт временный custom-format snapshot PostgreSQL, а затем восстанавливает его перед каждым прогревом и каждым измерением. В `finally` скрипт возвращает исходную БД и 2 API-реплики, поэтому все benchmark-записи удаляются, а данные, бывшие в volume до эксперимента, сохраняются. Все семь случаев (1/2/4 фиксированный бюджет; 1/2/4/8 рост) используют один и тот же snapshot. + +CI workflow `.github/workflows/application.yml` отдельно проверяет сборку Compose-стенда, публичный smoke, конкурентную идемпотентность, persistence и отказ одной реплики. CI использует только временные синтетические credentials и удаляет собственный тестовый volume после завершения job. + ## Что сделать 1. Выберите Compose + Nginx/HAProxy/Traefik либо Docker Swarm. Для выбранного варианта дайте одну последовательность запуска. diff --git a/REPORT.md b/REPORT.md index c2679005..2b2b7aba 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,113 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Аккуратов Тимофей, бвт2304 +- Целевая оценка: 5; выбран трек 1 — цена масштабирования +- Commit/PR: ещё не созданы; изменения находятся в локальном рабочем каталоге +- Источники, библиотеки и помощь ИИ: перенесена собственная реализация из mtusi2026lab2; Python stdlib, psycopg 3.2.10 и psycopg-pool 3.3.3; официальные образы PostgreSQL и HAProxy. При разработке использовалась помощь Copilot в VS Code; код и результаты проверены локально. Чужие исходники не переносились. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Desktop с Compose v2, `openssl`, `make` и Python 3.9+ для локальных проверок. API работает в образе Python 3.12.7; клиентские тесты используют только стандартную библиотеку. + +```sh +cp .env.example .env +./scripts/bootstrap-secrets.sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 120 + +make check +make test +make test-idempotency +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Для полной OpenAPI-валидации: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +`bootstrap-secrets.sh` создаёт локальные `secrets/postgres_password` и `secrets/database_url`, не печатая их. `.env` и `secrets/` игнорируются Git. Миграции применяются одноразовым сервисом `migrate`; дождитесь состояния `healthy` у PostgreSQL и API. По умолчанию тесты обращаются к `http://127.0.0.1:8080`; при изменении `APP_PORT` задайте, например, `make test BASE_URL=http://127.0.0.1:8081`. + +Безопасная остановка с сохранением данных: `docker compose --env-file .env down`. **Намеренное удаление всех учебных данных** — только отдельной командой `docker compose --env-file .env down -v`; эта команда не используется проверками restart/failover. + +Исследовательский эксперимент целиком повторяется одной командой после запуска Compose-стенда: + +```sh +make track1 +``` + +Команда запускает все семь случаев матрицы: 1/2/4 реплики при фиксированном бюджете и 1/2/4/8 при росте ресурсов. Для сопоставимости используйте тот же Docker host, порт, число клиентов и длительность. Экспериментальные записи удаляются при восстановлении сохранённого baseline. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Клиент → HAProxy → API × 2+ → общая PostgreSQL.** В Compose нет `container_name`; API-порты не опубликованы на хост. Единственный внешний порт — `127.0.0.1:${APP_PORT}:8080` у HAProxy. Реплики не используют sticky sessions. Сессии, сообщения и результаты идемпотентности хранятся в PostgreSQL; это единственный источник истины. Redis из предыдущей реализации остаётся необязательной возможностью в коде, но в этом стенде не развёрнут и для корректности не нужен. + +Из lab2 перенесены `src/`, `migrations/001_initial.sql`, Dockerfile и зависимости. Изменения этой работы: + +- API больше не запускает миграции при старте. Сервис `migrate` применяет версионированные SQL-файлы до запуска реплик; таблица `schema_migrations` и advisory transaction lock защищают от повторного применения. +- `migrations/002_idempotency.sql` добавляет общую таблицу ключей. Ключ уникален в области user + POST + chat + key; хранится SHA-256 канонического JSON и исходный ответ. Проверка повтора, создание сообщения и сохранение ответа выполняются в одной транзакции PostgreSQL. Transaction-level advisory lock сериализует конкурентные запросы с тем же ключом на разных репликах. Конфликтующий body возвращает 409, успешный повтор — исходное сообщение; срок хранения не меньше 24 часов. +- `X-Instance-Id` берётся из hostname контейнера и неизменен в пределах процесса. `scripts/stop-one.sh` сопоставляет его с конкретным контейнером. +- HAProxy обнаруживает backend через DNS Docker (`127.0.0.11`, `hold valid 2s`) и проверяет `GET /health/ready` каждую секунду; backend считается недоступным после двух неудач. С учётом DNS TTL и таймаута проверки консервативный бюджет обнаружения — до 5s, что ниже лимита 10s; connect timeout — 2s, client/server timeout — 15s. У API есть readiness healthcheck, лимиты БД: statement 5s, lock 2s, pool wait 3s. При SIGTERM сервер прекращает приём и завершает активные запросы; `stop_grace_period` — 20s. +- Все контейнерные образы закреплены тегами: Python 3.12.7-slim, PostgreSQL 16.4-alpine, HAProxy 3.0.7-alpine. PostgreSQL использует именованный `postgres_data`. API по умолчанию ограничен 1 CPU/512 MiB на реплику; PostgreSQL — 1 CPU/1 GiB, HAProxy — 0.5 CPU/128 MiB. В переменных `.env.example` настраиваются host port, TTL и per-replica лимиты API. + +Миграция 002 аддитивна и не меняет существующие таблицы lab2, поэтому старый API не использует новую таблицу. CI workflow `.github/workflows/application.yml` собирает Compose-стенд, запускает публичные и конкурентные тесты, persistence и отказ API-реплики; он не тестирует обновление приложения на новую версию бизнес-кода. Локально проверен поочерёдный restart одинакового образа под нагрузкой. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Контракт и OpenAPI | `make check`; `make validate PYTHON=.venv/bin/python` | PASS для структуры, синтаксиса и полной OpenAPI 3.1/JSON Schema валидации; контракт и публичные проверки не менялись | `evidence/openapi-validation.txt` | +| Стенд и миграции | `docker compose ... up --scale api=2`; SQL `schema_migrations` | 2 API healthy, PostgreSQL и HAProxy healthy; применены ровно `001_initial.sql`, `002_idempotency.sql`; API опубликованных host ports не имеет | `evidence/deployment-checks.txt` | +| Smoke и общая сессия | `make test` через HAProxy | PASS: полный публичный сценарий lab3 | `evidence/public-tests.txt` | +| Общая идемпотентность | `make test-idempotency` | 12 конкурентных запросов с одинаковым ключом прошли через 2 наблюдаемые реплики, вернули одно сообщение; в истории ровно одна запись. Публичный smoke также проверяет повтор и 409 при другом body | `evidence/public-tests.txt` | +| Persistence lab2 | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: после перезапуска PostgreSQL, API и LB подтверждённые данные доступны, прежняя сессия действительна; volumes сохранены | `evidence/persistence.txt` | +| Failover первой и второй API | Дважды `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, восстановление между прогонами | Проверки увидели ≥2 экземпляра. Для первого: 108 успешных записей, 2 неуспешные попытки, восстановление за 2.684s. Для второго: 109 успешных, 1 неуспешная, восстановление за 2.566s. Подтверждённые сообщения и сессия сохранились | `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/failover-targets.txt` | +| Readiness, timeouts, SIGTERM | HAProxy health checks и `scripts/rolling-restart.sh` | Backend исключается после 2 проверок с интервалом 1s; отказная проверка восстановила записи за 2.566–2.684s. Поочерёдный restart обоих контейнеров завершился `healthy` | `haproxy.cfg`, `compose.yaml`, `evidence/failover-1.txt`, `evidence/failover-2.txt`, `evidence/rolling-restart-actions.txt` | +| Лимиты и отсутствие обходного API-порта | `docker compose config`; `docker inspect` | Реально установлены API 1 CPU/512 MiB, PostgreSQL 1 CPU/1 GiB, HAProxy 0.5 CPU/128 MiB. На хост опубликован только loopback-порт LB; в `ps` у API показан только внутренний `8080/tcp` | `evidence/deployment-checks.txt` | +| Rolling restart под нагрузкой | `tests/load.py --duration 35 --concurrency 8` одновременно с `scripts/rolling-restart.sh` | 19,968 успешных POST, 6 ошибок переходного периода; successful RPS 570.303, p50/p95/p99 8.385/45.880/73.085 ms. Оба API контейнера были поочерёдно перезапущены и стали healthy | `evidence/rolling-restart-load.json`, `evidence/rolling-restart-actions.txt` | +| Трек 5 — цена масштабирования | `make track1` | Матрица 1/2/4 при фиксированных ресурсах, затем рост до 1/2/4/8; одна исходная custom-format копия PostgreSQL восстанавливается до каждого прогрева и измерения; три повтора на случай; исходная БД восстанавливается в `finally` | `evidence/track1/results.json`, `evidence/track1/throughput.svg` | +| CI сборка и проверка приложения | `.github/workflows/application.yml` (`push`, `pull_request`) | Отдельный job устанавливает requirements-dev, собирает/ожидает Compose, выполняет `make check validate`, smoke, параллельную идемпотентность, persistence и failover. Workflow добавлен; выполнение на GitHub Actions ожидает push/PR | `.github/workflows/application.yml` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Трек 1: цена масштабирования -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Гипотеза:** при фиксированном суммарном бюджете API увеличение числа реплик само по себе не даст прироста; при добавлении ресурсов оно ускорит запись до насыщения общей PostgreSQL, после чего новые API-процессы почти не помогут. + +Одна машина macOS 27.0.1 arm64, физически 10 CPU и 16 GiB RAM; Docker Desktop 28.0.4 / Compose 2.34.0-desktop.1 предоставлял контейнерам 10 vCPU и около 7.65 GiB. API: Python 3.12.7; фиксированные ресурсы PostgreSQL 1 CPU/1 GiB и HAProxy 0.5 CPU/128 MiB. На каждый случай — прогрев 5s и три измеряемых прогона по 30s, 16 параллельных клиентов. Генератор — штатный closed-loop `tests/load.py`, только POST сообщений через LB; прозрачных retry нет. Перед каждым прогревом и измеряемым прогоном API остановлен, PostgreSQL восстановлена из одного custom-format snapshot, проверены контрольные количества строк, затем запускается выбранное число реплик. После всей матрицы в `finally` возвращается исходная БД и две реплики; синтетические benchmark-записи не остаются в volume. Во время запуска исключены внешние записи в БД. В таблице latency — среднее p50/p95/p99 по успешным запросам, RPS — среднее ± стандартное отклонение между тремя прогонами. CPU — максимальная 1-секундная выборка Docker stats; 100% соответствует примерно одному CPU. + +| Профиль | API | Суммарный лимит API | Успешные RPS, среднее ± σ | p50 / p95 / p99, ms | Ошибки | Peak PostgreSQL CPU | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| Фиксированный бюджет | 1 | 1 CPU / 512 MiB | 212.54 ± 1.34 | 79.36 / 115.01 / 181.42 | 0 | 28.22% | +| Фиксированный бюджет | 2 | 1 CPU / 512 MiB | 212.71 ± 10.81 | 83.45 / 166.94 / 224.07 | 0 | 40.58% | +| Фиксированный бюджет | 4 | 1 CPU / 512 MiB | 156.34 ± 17.60 | 92.69 / 260.44 / 663.89 | 0 | 46.22% | +| Рост ресурсов | 1 | 1 CPU / 512 MiB | 211.67 ± 1.56 | 79.53 / 115.35 / 182.46 | 0 | 26.54% | +| Рост ресурсов | 2 | 2 CPU / 1 GiB | 604.95 ± 6.29 | 19.72 / 60.32 / 79.17 | 0 | 62.93% | +| Рост ресурсов | 4 | 4 CPU / 2 GiB | 1,099.73 ± 11.87 | 11.01 / 29.91 / 51.29 | 0 | 108.47% | +| Рост ресурсов | 8 | 8 CPU / 4 GiB | 1,159.95 ± 16.88 | 11.75 / 27.90 / 37.07 | 0 | 105.96% | + +При фиксированном бюджете переход к 4 маленьким репликам снизил успешный throughput на 26.4% относительно одной реплики и увеличил p95 с 115.01 до 260.44 ms: дробление CPU между процессами не дало выигрыша. При росте бюджета до четырёх API CPU/2 GiB throughput вырос в 5.19 раза относительно одной реплики при том же per-instance лимите, а p95 уменьшился примерно в 3.86 раза — подтверждённое улучшение для этого стенда. Дальнейшее удвоение API до 8, CPU до 8 и памяти до 4 GiB увеличило throughput на 5.48%; одновременно PostgreSQL достигла выборки 105.96–108.47% одного CPU. Это указывает на насыщение общей БД и уменьшающуюся отдачу от добавления API-ресурсов; это наблюдение относится только к данному профилю, а не универсальная характеристика PostgreSQL. + +Все данные синтетические. До матрицы был зафиксирован baseline snapshot; каждое измерение стартовало с одного и того же состояния БД и создавало свой тестовый набор в новом уникальном чате. Row counts после завершения совпали с baseline: users=156, chats=52, messages=445522, sessions=154, idempotency_records=445522. Пиковая выборка памяти за benchmark: PostgreSQL 252.6 MiB, API 73.1 MiB на контейнер, HAProxy 21.1 MiB. Подробные исходные метрики и выборки CPU/RAM, единый SHA-256 snapshot и контрольные количества строк сохранены в JSON. + +### Отказы + +| Воздействие | Ожидание | Наблюдение | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка первой API-реплики | LB продолжает читать/писать через оставшуюся реплику; повтор сообщения не создаёт дубль | 108 успешных записей, 2 переходные ошибки; readiness восстановилась за 2.684s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 | +| Остановка второй API-реплики | Те же гарантии для другой реплики | 109 успешных записей, 1 переходная ошибка; readiness восстановилась за 2.566s | Данные и сессия сохранены; потеря подтверждённых сообщений — 0 | +| Поочерёдный SIGTERM/restart двух API под нагрузкой | Вторая реплика остаётся доступной, остановленная возвращается healthy | 19,968 успешных POST, 6 ошибок за 35s; оба экземпляра перезапущены последовательно | Контейнеры вернулись в healthy; подтверждённые записи остаются в PostgreSQL | +| Restart PostgreSQL + API + LB | Сохранить существующую сессию и сообщения на named volume | Persistence-тест PASS с прежним токеном | Потеря данных — 0; volumes не удалялись | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Эксперимент закрытый (closed-loop), проводится на одном Mac и одним клиентским процессом; он подвержен coordinated omission и не задаёт независимый arrival rate. Процентили считаются только по успешным ответам, а выборка CPU/RAM снимается раз в секунду. +- Сценарий проверяет отказ API-процесса, не отказ Docker-host, HAProxy, PostgreSQL или диска. PostgreSQL и единственный LB — SPOF; отказ узла не исследовался, multi-node failover не заявляется. +- Порт HTTP открыт только на loopback для синтетического локального стенда. Для удалённой демонстрации необходим TLS. Секреты Compose — локальные bind-mounted файлы, не production secret manager. +- Идемпотентность ограничена текстовым контрактом lab3 и 24-часовым минимумом; транзакция держит PostgreSQL advisory lock. HAProxy `server-template` настроен максимум на 10 адресов — при масштабировании выше нужно расширить диапазон и повторить проверки. +- Rolling-сценарий проверяет поочерёдный restart текущего образа, не обновление на новую версию приложения. Миграции аддитивны, но совместимость произвольной смены версий приложения не проверялась. +- Результаты относятся только к указанным лимитам, размеру данных и клиентской интенсивности. Snapshot теперь одинаков перед каждым прогоном, но нагрузка остаётся closed-loop, на одном ноутбуке, без открытого arrival-rate профиля. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0ceb424f --- /dev/null +++ b/compose.yaml @@ -0,0 +1,105 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 2s + retries: 20 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 1G + restart: unless-stopped + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python3", "src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_POOL_MAX_SIZE: "2" + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX_SIZE: "8" + DB_POOL_TIMEOUT_SECONDS: "3" + DB_STATEMENT_TIMEOUT_MS: "5000" + DB_LOCK_TIMEOUT_MS: "2000" + secrets: + - database_url + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python3 + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()" + interval: 2s + timeout: 2s + retries: 3 + start_period: 10s + deploy: + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEMORY_LIMIT:-512M}" + stop_grace_period: 20s + restart: unless-stopped + + load_balancer: + image: haproxy:3.0.7-alpine + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + volumes: + - ./haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "haproxy -c -f /usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + +volumes: + postgres_data: diff --git a/evidence/deployment-checks.txt b/evidence/deployment-checks.txt new file mode 100644 index 00000000..0d8da876 --- /dev/null +++ b/evidence/deployment-checks.txt @@ -0,0 +1,16 @@ +Capture time: 2026-10-21T05:57:02Z +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-5 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab3-api-6 mtusi2026lab3-api "python3 src/server.…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi2026lab3-load_balancer-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" load_balancer 29 minutes ago Up 17 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab3-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 29 minutes ago Up 17 minutes (healthy) 5432/tcp + +Migration versions: +001_initial.sql +002_idempotency.sql + +Container limits: +/mtusi2026lab3-api-5 NanoCPUs=1000000000 MemoryBytes=536870912 +/mtusi2026lab3-api-6 NanoCPUs=1000000000 MemoryBytes=536870912 +/mtusi2026lab3-postgres-1 NanoCPUs=1000000000 MemoryBytes=1073741824 +/mtusi2026lab3-load_balancer-1 NanoCPUs=500000000 MemoryBytes=134217728 diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..4fc2f9b8 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 2, "first_recovery_seconds": 2.684} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..fe577933 --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.566} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-targets.txt b/evidence/failover-targets.txt new file mode 100644 index 00000000..55e111c6 --- /dev/null +++ b/evidence/failover-targets.txt @@ -0,0 +1,2 @@ +2026-10-21T05:39:45Z target_instance=498a08cd2b5c container=498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71 +2026-10-21T05:40:39Z target_instance=137c7f092d63 container=137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba diff --git a/evidence/openapi-validation.txt b/evidence/openapi-validation.txt new file mode 100644 index 00000000..f75669b2 --- /dev/null +++ b/evidence/openapi-validation.txt @@ -0,0 +1,4 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..90ac266a --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-tests.txt b/evidence/public-tests.txt new file mode 100644 index 00000000..8219a37d --- /dev/null +++ b/evidence/public-tests.txt @@ -0,0 +1,6 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://127.0.0.1:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +python3 student-tests/test_idempotency_race.py --base-url "http://127.0.0.1:8080" +PASS: 12 concurrent same-key requests on 2 replicas returned one message diff --git a/evidence/rolling-restart-actions.txt b/evidence/rolling-restart-actions.txt new file mode 100644 index 00000000..ac97f609 --- /dev/null +++ b/evidence/rolling-restart-actions.txt @@ -0,0 +1,29 @@ + Container mtusi2026lab3-postgres-1 Running + Container mtusi2026lab3-migrate-1 Created + Container mtusi2026lab3-api-3 Running + Container mtusi2026lab3-api-4 Created + Container mtusi2026lab3-load_balancer-1 Running + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-migrate-1 Starting + Container mtusi2026lab3-migrate-1 Started + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-api-4 Starting + Container mtusi2026lab3-api-4 Started + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-3 Healthy + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-load_balancer-1 Waiting + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-load_balancer-1 Healthy + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-3 Healthy +PASS: API container 498a08cd2b5cf94be02873ea9eb0c9c5f6b79133ddfd50b44aff134f86222b71 restarted and is healthy +PASS: API container 137c7f092d63cd2e0d8d2b71ceecd67860c35ba79709df56831bba3a967629ba restarted and is healthy diff --git a/evidence/rolling-restart-load.json b/evidence/rolling-restart-load.json new file mode 100644 index 00000000..92f641ef --- /dev/null +++ b/evidence/rolling-restart-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 35.0, + "elapsed_seconds": 35.013, + "concurrency": 8, + "successful_requests": 19968, + "failed_requests": 6, + "error_ratio": 0.0003, + "successful_rps": 570.303, + "successful_latency_ms": { + "p50": 8.385, + "p95": 45.88, + "p99": 73.085 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-1-run-1.json b/evidence/track1/fixed-1-run-1.json new file mode 100644 index 00000000..9e8f630d --- /dev/null +++ b/evidence/track1/fixed-1-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.046, + "concurrency": 16, + "successful_requests": 6346, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 211.212, + "successful_latency_ms": { + "p50": 80.264, + "p95": 114.896, + "p99": 182.694 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-1-run-2.json b/evidence/track1/fixed-1-run-2.json new file mode 100644 index 00000000..b271d8b0 --- /dev/null +++ b/evidence/track1/fixed-1-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.064, + "concurrency": 16, + "successful_requests": 6445, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 214.379, + "successful_latency_ms": { + "p50": 78.662, + "p95": 115.143, + "p99": 181.9 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-1-run-3.json b/evidence/track1/fixed-1-run-3.json new file mode 100644 index 00000000..cda87005 --- /dev/null +++ b/evidence/track1/fixed-1-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.056, + "concurrency": 16, + "successful_requests": 6373, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 212.038, + "successful_latency_ms": { + "p50": 79.158, + "p95": 114.991, + "p99": 179.677 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-1-warmup-run-0.json b/evidence/track1/fixed-1-warmup-run-0.json new file mode 100644 index 00000000..503376e4 --- /dev/null +++ b/evidence/track1/fixed-1-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.065, + "concurrency": 16, + "successful_requests": 1036, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 204.529, + "successful_latency_ms": { + "p50": 81.556, + "p95": 109.862, + "p99": 172.61 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-2-run-1.json b/evidence/track1/fixed-2-run-1.json new file mode 100644 index 00000000..27aa6a40 --- /dev/null +++ b/evidence/track1/fixed-2-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.098, + "concurrency": 16, + "successful_requests": 5968, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 198.285, + "successful_latency_ms": { + "p50": 86.571, + "p95": 190.715, + "p99": 278.78 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-2-run-2.json b/evidence/track1/fixed-2-run-2.json new file mode 100644 index 00000000..36d1fefd --- /dev/null +++ b/evidence/track1/fixed-2-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.03, + "concurrency": 16, + "successful_requests": 6736, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 224.312, + "successful_latency_ms": { + "p50": 80.141, + "p95": 150.189, + "p99": 197.092 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-2-run-3.json b/evidence/track1/fixed-2-run-3.json new file mode 100644 index 00000000..ad048586 --- /dev/null +++ b/evidence/track1/fixed-2-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 16, + "successful_requests": 6473, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 215.547, + "successful_latency_ms": { + "p50": 83.643, + "p95": 159.92, + "p99": 196.349 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-2-warmup-run-0.json b/evidence/track1/fixed-2-warmup-run-0.json new file mode 100644 index 00000000..8a772b5f --- /dev/null +++ b/evidence/track1/fixed-2-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.017, + "concurrency": 16, + "successful_requests": 990, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 197.311, + "successful_latency_ms": { + "p50": 82.99, + "p95": 199.803, + "p99": 294.885 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-4-run-1.json b/evidence/track1/fixed-4-run-1.json new file mode 100644 index 00000000..700c3901 --- /dev/null +++ b/evidence/track1/fixed-4-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.155, + "concurrency": 16, + "successful_requests": 5464, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 181.196, + "successful_latency_ms": { + "p50": 85.443, + "p95": 200.223, + "p99": 467.896 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-4-run-2.json b/evidence/track1/fixed-4-run-2.json new file mode 100644 index 00000000..9f281208 --- /dev/null +++ b/evidence/track1/fixed-4-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.201, + "concurrency": 16, + "successful_requests": 4377, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 144.927, + "successful_latency_ms": { + "p50": 98.238, + "p95": 287.157, + "p99": 494.409 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-4-run-3.json b/evidence/track1/fixed-4-run-3.json new file mode 100644 index 00000000..7206a678 --- /dev/null +++ b/evidence/track1/fixed-4-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.261, + "concurrency": 16, + "successful_requests": 4324, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 142.888, + "successful_latency_ms": { + "p50": 94.377, + "p95": 293.944, + "p99": 1029.361 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/fixed-4-warmup-run-0.json b/evidence/track1/fixed-4-warmup-run-0.json new file mode 100644 index 00000000..2e7bf065 --- /dev/null +++ b/evidence/track1/fixed-4-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.173, + "concurrency": 16, + "successful_requests": 1140, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 220.377, + "successful_latency_ms": { + "p50": 64.745, + "p95": 197.524, + "p99": 398.349 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-1-run-1.json b/evidence/track1/growth-1-run-1.json new file mode 100644 index 00000000..4097620b --- /dev/null +++ b/evidence/track1/growth-1-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.039, + "concurrency": 16, + "successful_requests": 6333, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 210.824, + "successful_latency_ms": { + "p50": 80.103, + "p95": 114.112, + "p99": 182.734 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-1-run-2.json b/evidence/track1/growth-1-run-2.json new file mode 100644 index 00000000..66983e30 --- /dev/null +++ b/evidence/track1/growth-1-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.073, + "concurrency": 16, + "successful_requests": 6325, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 210.32, + "successful_latency_ms": { + "p50": 79.313, + "p95": 117.188, + "p99": 183.607 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-1-run-3.json b/evidence/track1/growth-1-run-3.json new file mode 100644 index 00000000..6581c6dc --- /dev/null +++ b/evidence/track1/growth-1-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.03, + "concurrency": 16, + "successful_requests": 6422, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 213.855, + "successful_latency_ms": { + "p50": 79.164, + "p95": 114.738, + "p99": 181.052 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-1-warmup-run-0.json b/evidence/track1/growth-1-warmup-run-0.json new file mode 100644 index 00000000..62d5fc11 --- /dev/null +++ b/evidence/track1/growth-1-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.03, + "concurrency": 16, + "successful_requests": 1054, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 209.531, + "successful_latency_ms": { + "p50": 77.196, + "p95": 116.562, + "p99": 184.922 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-2-run-1.json b/evidence/track1/growth-2-run-1.json new file mode 100644 index 00000000..d5413e7b --- /dev/null +++ b/evidence/track1/growth-2-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.017, + "concurrency": 16, + "successful_requests": 18059, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 601.627, + "successful_latency_ms": { + "p50": 19.831, + "p95": 60.095, + "p99": 81.757 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-2-run-2.json b/evidence/track1/growth-2-run-2.json new file mode 100644 index 00000000..219ccbfe --- /dev/null +++ b/evidence/track1/growth-2-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.494, + "concurrency": 16, + "successful_requests": 18280, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 599.463, + "successful_latency_ms": { + "p50": 19.535, + "p95": 59.859, + "p99": 78.355 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-2-run-3.json b/evidence/track1/growth-2-run-3.json new file mode 100644 index 00000000..d0a5ee7e --- /dev/null +++ b/evidence/track1/growth-2-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.029, + "concurrency": 16, + "successful_requests": 18431, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 613.766, + "successful_latency_ms": { + "p50": 19.791, + "p95": 61.005, + "p99": 77.388 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-2-warmup-run-0.json b/evidence/track1/growth-2-warmup-run-0.json new file mode 100644 index 00000000..a84f1fc4 --- /dev/null +++ b/evidence/track1/growth-2-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.037, + "concurrency": 16, + "successful_requests": 3029, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 601.302, + "successful_latency_ms": { + "p50": 20.938, + "p95": 59.639, + "p99": 76.915 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-4-run-1.json b/evidence/track1/growth-4-run-1.json new file mode 100644 index 00000000..94a68db5 --- /dev/null +++ b/evidence/track1/growth-4-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.012, + "concurrency": 16, + "successful_requests": 32846, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1094.415, + "successful_latency_ms": { + "p50": 10.732, + "p95": 30.846, + "p99": 51.188 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-4-run-2.json b/evidence/track1/growth-4-run-2.json new file mode 100644 index 00000000..bf6061bd --- /dev/null +++ b/evidence/track1/growth-4-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.013, + "concurrency": 16, + "successful_requests": 32672, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1088.609, + "successful_latency_ms": { + "p50": 11.145, + "p95": 30.074, + "p99": 53.423 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-4-run-3.json b/evidence/track1/growth-4-run-3.json new file mode 100644 index 00000000..88a86ae2 --- /dev/null +++ b/evidence/track1/growth-4-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 16, + "successful_requests": 33502, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1116.177, + "successful_latency_ms": { + "p50": 11.142, + "p95": 28.815, + "p99": 49.26 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-4-warmup-run-0.json b/evidence/track1/growth-4-warmup-run-0.json new file mode 100644 index 00000000..94894afd --- /dev/null +++ b/evidence/track1/growth-4-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.014, + "concurrency": 16, + "successful_requests": 5472, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1091.286, + "successful_latency_ms": { + "p50": 11.367, + "p95": 33.413, + "p99": 54.849 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-8-run-1.json b/evidence/track1/growth-8-run-1.json new file mode 100644 index 00000000..eab3e079 --- /dev/null +++ b/evidence/track1/growth-8-run-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.033, + "concurrency": 16, + "successful_requests": 34199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1138.726, + "successful_latency_ms": { + "p50": 11.784, + "p95": 29.272, + "p99": 38.508 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-8-run-2.json b/evidence/track1/growth-8-run-2.json new file mode 100644 index 00000000..fb0df2b6 --- /dev/null +++ b/evidence/track1/growth-8-run-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 16, + "successful_requests": 35418, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1180.016, + "successful_latency_ms": { + "p50": 11.711, + "p95": 26.361, + "p99": 36.002 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-8-run-3.json b/evidence/track1/growth-8-run-3.json new file mode 100644 index 00000000..392cbb4e --- /dev/null +++ b/evidence/track1/growth-8-run-3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.012, + "concurrency": 16, + "successful_requests": 34848, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1161.122, + "successful_latency_ms": { + "p50": 11.745, + "p95": 28.074, + "p99": 36.688 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/growth-8-warmup-run-0.json b/evidence/track1/growth-8-warmup-run-0.json new file mode 100644 index 00000000..06ad484f --- /dev/null +++ b/evidence/track1/growth-8-warmup-run-0.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 5.0, + "elapsed_seconds": 5.014, + "concurrency": 16, + "successful_requests": 5457, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1088.404, + "successful_latency_ms": { + "p50": 11.999, + "p95": 31.015, + "p99": 44.748 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/results.json b/evidence/track1/results.json new file mode 100644 index 00000000..7924f631 --- /dev/null +++ b/evidence/track1/results.json @@ -0,0 +1,6529 @@ +{ + "method": "closed-loop POST message workload via HAProxy; same concurrency and per-run restored PostgreSQL custom-format baseline snapshot", + "database_baseline": { + "snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "restore_before_each_warmup_and_measured_repetition": true, + "state_after_experiment": "baseline restored; all synthetic benchmark writes removed" + }, + "system": { + "platform": "macOS-27.0.1-arm64-arm-64bit-Mach-O", + "processor": "arm", + "cpu_count": 10, + "host_memory_bytes": "17179869184", + "docker_server": "28.0.4", + "docker_compose": "2.34.0-desktop.1" + }, + "database_and_lb_limits": { + "postgres": "1 CPU / 1 GiB", + "HAProxy": "0.5 CPU / 128 MiB" + }, + "client_concurrency": 16, + "requested_duration_seconds": 30, + "repetitions_per_case": 3, + "cases": [ + { + "phase": "fixed", + "api_replicas": 1, + "per_replica_cpu_limit": "1.0000", + "per_replica_memory_limit": "512M", + "total_api_cpu_limit": 1.0, + "total_api_memory_limit_mib": 512, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.046, + "concurrency": 16, + "successful_requests": 6346, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 211.212, + "successful_latency_ms": { + "p50": 80.264, + "p95": 114.896, + "p99": 182.694 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.41, + "memory_bytes": 44386222 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.41, + "memory_bytes": 15917384 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.81, + "memory_bytes": 121844531 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.79, + "memory_bytes": 31415337 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.91, + "memory_bytes": 15351153 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.84, + "memory_bytes": 122054246 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.63, + "memory_bytes": 31855739 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.41, + "memory_bytes": 15560868 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.76, + "memory_bytes": 122473677 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.13, + "memory_bytes": 31939625 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.7, + "memory_bytes": 15560868 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.76, + "memory_bytes": 126038835 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.55, + "memory_bytes": 32138854 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.71, + "memory_bytes": 15697183 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.43, + "memory_bytes": 127401984 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.57, + "memory_bytes": 32495370 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.3, + "memory_bytes": 15843983 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.47, + "memory_bytes": 128240845 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.21, + "memory_bytes": 31614566 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.61, + "memory_bytes": 15969812 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.61, + "memory_bytes": 129499136 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.53, + "memory_bytes": 32222740 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.28, + "memory_bytes": 16777216 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.22, + "memory_bytes": 130547712 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.0, + "memory_bytes": 31939625 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.29, + "memory_bytes": 16315843 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.88, + "memory_bytes": 131596288 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.064, + "concurrency": 16, + "successful_requests": 6445, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 214.379, + "successful_latency_ms": { + "p50": 78.662, + "p95": 115.143, + "p99": 181.9 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.45, + "memory_bytes": 30838620 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.02, + "memory_bytes": 16294871 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 20.19, + "memory_bytes": 132435149 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.67, + "memory_bytes": 31446794 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.38, + "memory_bytes": 16158556 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.72, + "memory_bytes": 132749722 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.87, + "memory_bytes": 31782339 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.14, + "memory_bytes": 16231956 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.58, + "memory_bytes": 133378867 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 108.29, + "memory_bytes": 44291850 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.36, + "memory_bytes": 16494100 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.4, + "memory_bytes": 133274010 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.49, + "memory_bytes": 45455770 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.2, + "memory_bytes": 16263414 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.26, + "memory_bytes": 133798298 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.41, + "memory_bytes": 32044483 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.99, + "memory_bytes": 16913531 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.5, + "memory_bytes": 133064294 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.0, + "memory_bytes": 31834767 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.72, + "memory_bytes": 16347300 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.33, + "memory_bytes": 133588582 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.36, + "memory_bytes": 31855739 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.34, + "memory_bytes": 16651387 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.81, + "memory_bytes": 133588582 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.91, + "memory_bytes": 34844180 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.5, + "memory_bytes": 16483615 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.93, + "memory_bytes": 133064294 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.056, + "concurrency": 16, + "successful_requests": 6373, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 212.038, + "successful_latency_ms": { + "p50": 79.158, + "p95": 114.991, + "p99": 179.677 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.32, + "memory_bytes": 44627395 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.59, + "memory_bytes": 16315843 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.7, + "memory_bytes": 132644864 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.7, + "memory_bytes": 31478252 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.68, + "memory_bytes": 16651387 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 28.22, + "memory_bytes": 137363456 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.53, + "memory_bytes": 31509709 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.37, + "memory_bytes": 16399729 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.06, + "memory_bytes": 137468314 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.48, + "memory_bytes": 32243712 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.76, + "memory_bytes": 16965960 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 24.07, + "memory_bytes": 153511526 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.04, + "memory_bytes": 32621199 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.31, + "memory_bytes": 16441672 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.74, + "memory_bytes": 133903155 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.63, + "memory_bytes": 32453427 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.32, + "memory_bytes": 16787702 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.58, + "memory_bytes": 134112870 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.35, + "memory_bytes": 32243712 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.47, + "memory_bytes": 16515072 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.35, + "memory_bytes": 134951731 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 102.85, + "memory_bytes": 32023511 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.3, + "memory_bytes": 16494100 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.9, + "memory_bytes": 134112870 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.71, + "memory_bytes": 32537313 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.8, + "memory_bytes": 16567501 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.9, + "memory_bytes": 133903155 + } + ] + } + ] + }, + { + "phase": "fixed", + "api_replicas": 2, + "per_replica_cpu_limit": "0.5000", + "per_replica_memory_limit": "256M", + "total_api_cpu_limit": 1.0, + "total_api_memory_limit_mib": 512, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.098, + "concurrency": 16, + "successful_requests": 5968, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 198.285, + "successful_latency_ms": { + "p50": 86.571, + "p95": 190.715, + "p99": 278.78 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 40.21, + "memory_bytes": 41219523 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 50.72, + "memory_bytes": 42456842 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.18, + "memory_bytes": 16913531 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 28.94, + "memory_bytes": 165675008 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.81, + "memory_bytes": 30859592 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 43.57, + "memory_bytes": 30943478 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.11, + "memory_bytes": 16473129 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 23.74, + "memory_bytes": 166723584 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.66, + "memory_bytes": 30932992 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 50.94, + "memory_bytes": 31037850 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.18, + "memory_bytes": 16483615 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 20.38, + "memory_bytes": 163892429 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.22, + "memory_bytes": 31142707 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.46, + "memory_bytes": 30849106 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.6, + "memory_bytes": 16756244 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.27, + "memory_bytes": 163997286 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.17, + "memory_bytes": 31111250 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 44.4, + "memory_bytes": 43085988 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.62, + "memory_bytes": 16462643 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.76, + "memory_bytes": 163997286 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 50.91, + "memory_bytes": 41617981 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 43.28, + "memory_bytes": 42698015 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.92, + "memory_bytes": 16462643 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 23.41, + "memory_bytes": 163892429 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 43.97, + "memory_bytes": 31027364 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.59, + "memory_bytes": 40779121 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.97, + "memory_bytes": 16997417 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 20.95, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 29.65, + "memory_bytes": 31299994 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.34, + "memory_bytes": 30974935 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.25, + "memory_bytes": 16504586 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.44, + "memory_bytes": 164941005 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 31.2, + "memory_bytes": 30912020 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.33, + "memory_bytes": 30974935 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.24, + "memory_bytes": 16724787 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 25.14, + "memory_bytes": 163892429 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.03, + "concurrency": 16, + "successful_requests": 6736, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 224.312, + "successful_latency_ms": { + "p50": 80.141, + "p95": 150.189, + "p99": 197.092 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.29, + "memory_bytes": 31216108 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 53.68, + "memory_bytes": 42005955 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.95, + "memory_bytes": 16944988 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.04, + "memory_bytes": 157496115 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.25, + "memory_bytes": 30786191 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 40.69, + "memory_bytes": 43861934 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.79, + "memory_bytes": 16462643 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.13, + "memory_bytes": 162424422 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.64, + "memory_bytes": 31142707 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.63, + "memory_bytes": 31006392 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.53, + "memory_bytes": 16840131 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.67, + "memory_bytes": 162529280 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 37.09, + "memory_bytes": 40149975 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.77, + "memory_bytes": 30985421 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.82, + "memory_bytes": 16515072 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 29.1, + "memory_bytes": 162738995 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 43.23, + "memory_bytes": 30922506 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.12, + "memory_bytes": 42708500 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.14, + "memory_bytes": 16494100 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 26.3, + "memory_bytes": 164836147 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 45.91, + "memory_bytes": 31069307 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 53.0, + "memory_bytes": 31226593 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 13.39, + "memory_bytes": 16640901 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 32.68, + "memory_bytes": 164836147 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 40.99, + "memory_bytes": 41638953 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 49.13, + "memory_bytes": 30838620 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 13.53, + "memory_bytes": 16850616 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 40.58, + "memory_bytes": 163787571 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 40.72, + "memory_bytes": 30922506 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 52.01, + "memory_bytes": 31425823 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.74, + "memory_bytes": 16536044 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.56, + "memory_bytes": 164207002 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 40.73, + "memory_bytes": 31100764 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 51.4, + "memory_bytes": 31037850 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.79, + "memory_bytes": 16630415 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 24.28, + "memory_bytes": 164102144 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 16, + "successful_requests": 6473, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 215.547, + "successful_latency_ms": { + "p50": 83.643, + "p95": 159.92, + "p99": 196.349 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 53.87, + "memory_bytes": 40747663 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 29.41, + "memory_bytes": 40328233 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.11, + "memory_bytes": 16724787 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 31.49, + "memory_bytes": 159383552 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.43, + "memory_bytes": 31593595 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 31.02, + "memory_bytes": 30754734 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.1, + "memory_bytes": 16536044 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.6, + "memory_bytes": 162634138 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.56, + "memory_bytes": 31016878 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 32.61, + "memory_bytes": 40821064 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.42, + "memory_bytes": 17658020 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.51, + "memory_bytes": 163263283 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 42.67, + "memory_bytes": 30974935 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 45.26, + "memory_bytes": 43987763 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.46, + "memory_bytes": 16630415 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 22.52, + "memory_bytes": 162948710 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.54, + "memory_bytes": 31048335 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.11, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.23, + "memory_bytes": 16766730 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.95, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.83, + "memory_bytes": 31132221 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 31.76, + "memory_bytes": 31247565 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.1, + "memory_bytes": 16882074 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 23.06, + "memory_bytes": 163053568 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 47.45, + "memory_bytes": 31100764 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 48.33, + "memory_bytes": 41513124 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.3, + "memory_bytes": 16903045 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 27.01, + "memory_bytes": 163053568 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.12, + "memory_bytes": 31394365 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 41.58, + "memory_bytes": 30796677 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.88, + "memory_bytes": 16546529 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 20.89, + "memory_bytes": 163577856 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 52.57, + "memory_bytes": 31163679 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 34.39, + "memory_bytes": 30922506 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.46, + "memory_bytes": 16609444 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.53, + "memory_bytes": 163158426 + } + ] + } + ] + }, + { + "phase": "fixed", + "api_replicas": 4, + "per_replica_cpu_limit": "0.2500", + "per_replica_memory_limit": "128M", + "total_api_cpu_limit": 1.0, + "total_api_memory_limit_mib": 512, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.155, + "concurrency": 16, + "successful_requests": 5464, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 181.196, + "successful_latency_ms": { + "p50": 85.443, + "p95": 200.223, + "p99": 467.896 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 12.42, + "memory_bytes": 29695672 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 0.15, + "memory_bytes": 29643244 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 0.17, + "memory_bytes": 29632758 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 15.82, + "memory_bytes": 72320287 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 0.52, + "memory_bytes": 16326328 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 2.19, + "memory_bytes": 130442854 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 25.05, + "memory_bytes": 30188503 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 21.48, + "memory_bytes": 42635100 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 26.2, + "memory_bytes": 43589304 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 23.37, + "memory_bytes": 42582671 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.14, + "memory_bytes": 16525558 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.23, + "memory_bytes": 182032794 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 23.45, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.52, + "memory_bytes": 31037850 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 17.57, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 17.52, + "memory_bytes": 30157046 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.44, + "memory_bytes": 16577987 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.63, + "memory_bytes": 190001971 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 18.0, + "memory_bytes": 30136074 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.91, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 16.59, + "memory_bytes": 30377247 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 17.88, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 12.59, + "memory_bytes": 16819159 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.94, + "memory_bytes": 189897114 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 20.35, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.54, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 22.51, + "memory_bytes": 40191918 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 9.06, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 14.04, + "memory_bytes": 16819159 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 38.69, + "memory_bytes": 190001971 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 14.95, + "memory_bytes": 30209475 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 18.97, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 20.36, + "memory_bytes": 40516977 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 24.12, + "memory_bytes": 41313894 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.78, + "memory_bytes": 16598958 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 46.22, + "memory_bytes": 197971149 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 18.17, + "memory_bytes": 41030779 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 12.88, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 8.11, + "memory_bytes": 30649876 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.48, + "memory_bytes": 41848668 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.97, + "memory_bytes": 16494100 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 28.83, + "memory_bytes": 200907162 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 19.07, + "memory_bytes": 41209037 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 10.59, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 13.06, + "memory_bytes": 40129004 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.15, + "memory_bytes": 43253760 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.25, + "memory_bytes": 16504586 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 37.64, + "memory_bytes": 216426086 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 6.49, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 22.38, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.49, + "memory_bytes": 44061164 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.15, + "memory_bytes": 43306189 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.69, + "memory_bytes": 16756244 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.6, + "memory_bytes": 199439155 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 4.25, + "memory_bytes": 30765220 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 16.68, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 17.55, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 16.72, + "memory_bytes": 40527462 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 3.55, + "memory_bytes": 16399729 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 11.13, + "memory_bytes": 199439155 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.201, + "concurrency": 16, + "successful_requests": 4377, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 144.927, + "successful_latency_ms": { + "p50": 98.238, + "p95": 287.157, + "p99": 494.409 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 23.13, + "memory_bytes": 43117445 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 13.05, + "memory_bytes": 39552287 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 12.65, + "memory_bytes": 41544581 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.02, + "memory_bytes": 76619448 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 0.43, + "memory_bytes": 16315843 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 0.93, + "memory_bytes": 130547712 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 17.47, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 16.24, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 15.05, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 16.74, + "memory_bytes": 30157046 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.92, + "memory_bytes": 16798188 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.28, + "memory_bytes": 189163110 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 16.54, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.11, + "memory_bytes": 40307261 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 15.55, + "memory_bytes": 30188503 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 17.33, + "memory_bytes": 30083645 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.94, + "memory_bytes": 16515072 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.85, + "memory_bytes": 191889408 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 18.13, + "memory_bytes": 30649876 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.98, + "memory_bytes": 40024146 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 17.0, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 22.16, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 12.32, + "memory_bytes": 16609444 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.6, + "memory_bytes": 195978854 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 20.05, + "memory_bytes": 39919288 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.79, + "memory_bytes": 40894464 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.0, + "memory_bytes": 40737178 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 19.46, + "memory_bytes": 30461133 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.49, + "memory_bytes": 16598958 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 27.42, + "memory_bytes": 197132288 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 11.25, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 22.32, + "memory_bytes": 40988836 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 22.45, + "memory_bytes": 40925921 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 16.95, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.0, + "memory_bytes": 16483615 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 23.67, + "memory_bytes": 197132288 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 8.73, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.54, + "memory_bytes": 42561700 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.4, + "memory_bytes": 42142269 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 18.1, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.23, + "memory_bytes": 16557015 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.74, + "memory_bytes": 197237146 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 8.16, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 27.01, + "memory_bytes": 43778048 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.6, + "memory_bytes": 43400561 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.57, + "memory_bytes": 43683676 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.47, + "memory_bytes": 16525558 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.45, + "memory_bytes": 198285722 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 15.36, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 22.56, + "memory_bytes": 43012588 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.34, + "memory_bytes": 43358618 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 22.28, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.39, + "memory_bytes": 16494100 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 24.97, + "memory_bytes": 199753728 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 1.63, + "memory_bytes": 30408704 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 5.01, + "memory_bytes": 30555505 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 13.21, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 4.42, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 0.16, + "memory_bytes": 16431186 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 8.62, + "memory_bytes": 199648870 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.261, + "concurrency": 16, + "successful_requests": 4324, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 142.888, + "successful_latency_ms": { + "p50": 94.377, + "p95": 293.944, + "p99": 1029.361 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 24.82, + "memory_bytes": 42698015 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 0.18, + "memory_bytes": 29632758 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 12.6, + "memory_bytes": 29664215 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 19.12, + "memory_bytes": 42739958 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 0.25, + "memory_bytes": 16315843 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 1.51, + "memory_bytes": 130862285 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 17.92, + "memory_bytes": 30073160 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 17.72, + "memory_bytes": 30031217 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.12, + "memory_bytes": 42844815 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 18.13, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.62, + "memory_bytes": 16557015 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 38.93, + "memory_bytes": 179725926 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 22.4, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.22, + "memory_bytes": 30136074 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 10.54, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.0, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.79, + "memory_bytes": 16777216 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 25.16, + "memory_bytes": 191994266 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 15.87, + "memory_bytes": 30408704 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.78, + "memory_bytes": 30775706 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 15.84, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 24.7, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.96, + "memory_bytes": 16577987 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 34.16, + "memory_bytes": 195139994 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 9.7, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 4.04, + "memory_bytes": 30219960 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 9.25, + "memory_bytes": 40317747 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 24.68, + "memory_bytes": 30786191 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 3.89, + "memory_bytes": 16525558 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 10.99, + "memory_bytes": 194825421 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 14.09, + "memory_bytes": 40674263 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 9.8, + "memory_bytes": 30429676 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 14.59, + "memory_bytes": 40883978 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 26.4, + "memory_bytes": 43819991 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.55, + "memory_bytes": 16567501 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.84, + "memory_bytes": 199858586 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 23.45, + "memory_bytes": 30471619 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 10.46, + "memory_bytes": 39950746 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.3, + "memory_bytes": 43620762 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 22.69, + "memory_bytes": 30838620 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.75, + "memory_bytes": 16536044 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 33.78, + "memory_bytes": 201955738 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 15.97, + "memory_bytes": 30377247 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 9.64, + "memory_bytes": 30471619 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.4, + "memory_bytes": 43788534 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 18.75, + "memory_bytes": 30807163 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.05, + "memory_bytes": 16567501 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.53, + "memory_bytes": 203109171 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 16.37, + "memory_bytes": 30419190 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 9.67, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 20.58, + "memory_bytes": 41009807 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 21.43, + "memory_bytes": 30628905 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.21, + "memory_bytes": 16536044 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 22.25, + "memory_bytes": 202165453 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 25.09, + "memory_bytes": 43683676 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 8.03, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 7.95, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 21.67, + "memory_bytes": 30702305 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 5.85, + "memory_bytes": 16525558 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.66, + "memory_bytes": 203004314 + } + ] + } + ] + }, + { + "phase": "growth", + "api_replicas": 1, + "per_replica_cpu_limit": "1.0", + "per_replica_memory_limit": "512M", + "total_api_cpu_limit": 1.0, + "total_api_memory_limit_mib": 512, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.039, + "concurrency": 16, + "successful_requests": 6333, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 210.824, + "successful_latency_ms": { + "p50": 80.103, + "p95": 114.112, + "p99": 182.734 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.25, + "memory_bytes": 44166021 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.37, + "memory_bytes": 16557015 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 20.9, + "memory_bytes": 143969485 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.59, + "memory_bytes": 31520195 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.61, + "memory_bytes": 22072525 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.27, + "memory_bytes": 143759770 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.71, + "memory_bytes": 31604081 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.37, + "memory_bytes": 16913531 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.31, + "memory_bytes": 143864627 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.44, + "memory_bytes": 31813796 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.98, + "memory_bytes": 16630415 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.17, + "memory_bytes": 143864627 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 108.41, + "memory_bytes": 31992054 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.68, + "memory_bytes": 16619930 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.69, + "memory_bytes": 144913203 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.53, + "memory_bytes": 32621199 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.86, + "memory_bytes": 16693330 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.36, + "memory_bytes": 144179200 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.97, + "memory_bytes": 31771853 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.79, + "memory_bytes": 16630415 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.72, + "memory_bytes": 144703488 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.7, + "memory_bytes": 32285655 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.24, + "memory_bytes": 16924017 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.31, + "memory_bytes": 144493773 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.83, + "memory_bytes": 32222740 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.46, + "memory_bytes": 16724787 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 21.24, + "memory_bytes": 148163789 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.073, + "concurrency": 16, + "successful_requests": 6325, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 210.32, + "successful_latency_ms": { + "p50": 79.313, + "p95": 117.188, + "p99": 183.607 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.33, + "memory_bytes": 44019220 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 6.15, + "memory_bytes": 16588472 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.03, + "memory_bytes": 143445197 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.84, + "memory_bytes": 31666995 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.12, + "memory_bytes": 16745759 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.18, + "memory_bytes": 144179200 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.22, + "memory_bytes": 37455135 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 11.27, + "memory_bytes": 16735273 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.3, + "memory_bytes": 143969485 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.43, + "memory_bytes": 31939625 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.68, + "memory_bytes": 16672358 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.0, + "memory_bytes": 144493773 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.94, + "memory_bytes": 32338084 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.09, + "memory_bytes": 16787702 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.61, + "memory_bytes": 143759770 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.07, + "memory_bytes": 31698452 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.65, + "memory_bytes": 16777216 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.38, + "memory_bytes": 143864627 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.38, + "memory_bytes": 32254198 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.28, + "memory_bytes": 16682844 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.4, + "memory_bytes": 143864627 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.09, + "memory_bytes": 31981568 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.67, + "memory_bytes": 17574134 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.81, + "memory_bytes": 144388915 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.3, + "memory_bytes": 32327598 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.44, + "memory_bytes": 16808673 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 14.38, + "memory_bytes": 144388915 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.03, + "concurrency": 16, + "successful_requests": 6422, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 213.855, + "successful_latency_ms": { + "p50": 79.164, + "p95": 114.738, + "p99": 181.052 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 102.67, + "memory_bytes": 30555505 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.68, + "memory_bytes": 16682844 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 19.21, + "memory_bytes": 148373504 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 105.57, + "memory_bytes": 31646024 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.49, + "memory_bytes": 16714301 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 26.54, + "memory_bytes": 163787571 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 108.53, + "memory_bytes": 32296141 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.79, + "memory_bytes": 17773363 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.2, + "memory_bytes": 145332634 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.46, + "memory_bytes": 45015368 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 8.12, + "memory_bytes": 17028874 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.18, + "memory_bytes": 145437491 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.84, + "memory_bytes": 31792824 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.29, + "memory_bytes": 17553162 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.0, + "memory_bytes": 145227776 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.81, + "memory_bytes": 32537313 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 9.17, + "memory_bytes": 16766730 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 17.03, + "memory_bytes": 145332634 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.08, + "memory_bytes": 45141197 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.97, + "memory_bytes": 18161336 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 18.93, + "memory_bytes": 145856922 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.82, + "memory_bytes": 31666995 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 10.31, + "memory_bytes": 17081303 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 16.33, + "memory_bytes": 145122918 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.32, + "memory_bytes": 31666995 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 7.88, + "memory_bytes": 17102275 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 15.49, + "memory_bytes": 145332634 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 0.33, + "memory_bytes": 31174164 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 0.29, + "memory_bytes": 16630415 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 11.58, + "memory_bytes": 145332634 + } + ] + } + ] + }, + { + "phase": "growth", + "api_replicas": 2, + "per_replica_cpu_limit": "1.0", + "per_replica_memory_limit": "512M", + "total_api_cpu_limit": 2.0, + "total_api_memory_limit_mib": 1024, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.017, + "concurrency": 16, + "successful_requests": 18059, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 601.627, + "successful_latency_ms": { + "p50": 19.831, + "p95": 60.095, + "p99": 81.757 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 91.35, + "memory_bytes": 40663777 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 92.98, + "memory_bytes": 31247565 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 24.12, + "memory_bytes": 16798188 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 62.93, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 102.61, + "memory_bytes": 31247565 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 74.13, + "memory_bytes": 31205622 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.11, + "memory_bytes": 16861102 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 47.09, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 79.02, + "memory_bytes": 31153193 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 103.24, + "memory_bytes": 31132221 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 21.61, + "memory_bytes": 16861102 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 47.27, + "memory_bytes": 163892429 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 100.26, + "memory_bytes": 30817649 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 88.6, + "memory_bytes": 31142707 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 21.68, + "memory_bytes": 17186161 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 51.28, + "memory_bytes": 164311859 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 75.64, + "memory_bytes": 44186993 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 105.68, + "memory_bytes": 31635538 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 21.65, + "memory_bytes": 17448305 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 49.59, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 88.46, + "memory_bytes": 30859592 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 94.91, + "memory_bytes": 31341937 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 21.84, + "memory_bytes": 16913531 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 56.26, + "memory_bytes": 164626432 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 78.45, + "memory_bytes": 40789606 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 100.03, + "memory_bytes": 31153193 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.96, + "memory_bytes": 16934502 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 55.44, + "memory_bytes": 164207002 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 53.27, + "memory_bytes": 30985421 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 106.49, + "memory_bytes": 31436308 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 20.32, + "memory_bytes": 16997417 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 45.23, + "memory_bytes": 164416717 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 86.17, + "memory_bytes": 31027364 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 91.68, + "memory_bytes": 44910510 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 20.51, + "memory_bytes": 17553162 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 56.59, + "memory_bytes": 164521574 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.494, + "concurrency": 16, + "successful_requests": 18280, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 599.463, + "successful_latency_ms": { + "p50": 19.535, + "p95": 59.859, + "p99": 78.355 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.74, + "memory_bytes": 30932992 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 97.54, + "memory_bytes": 30765220 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 18.91, + "memory_bytes": 17532191 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 45.95, + "memory_bytes": 164102144 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 108.24, + "memory_bytes": 31090278 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 59.04, + "memory_bytes": 30817649 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 22.04, + "memory_bytes": 17479762 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 53.65, + "memory_bytes": 164521574 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 109.3, + "memory_bytes": 44438651 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 74.75, + "memory_bytes": 31289508 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 21.0, + "memory_bytes": 17039360 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 52.43, + "memory_bytes": 169030451 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 64.47, + "memory_bytes": 30912020 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 106.89, + "memory_bytes": 31016878 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 22.12, + "memory_bytes": 17102275 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 52.88, + "memory_bytes": 169135309 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 106.27, + "memory_bytes": 31027364 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 75.34, + "memory_bytes": 30765220 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 25.54, + "memory_bytes": 17846764 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 45.93, + "memory_bytes": 165465293 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 109.17, + "memory_bytes": 31446794 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 61.92, + "memory_bytes": 30775706 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 19.91, + "memory_bytes": 17395876 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 50.94, + "memory_bytes": 165989581 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 84.28, + "memory_bytes": 31782339 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 104.59, + "memory_bytes": 30786191 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.9, + "memory_bytes": 17081303 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 50.19, + "memory_bytes": 166094438 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.32, + "memory_bytes": 31205622 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 58.19, + "memory_bytes": 31016878 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.44, + "memory_bytes": 17154703 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 45.47, + "memory_bytes": 165884723 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.27, + "memory_bytes": 31016878 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 78.94, + "memory_bytes": 44417679 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.98, + "memory_bytes": 17175675 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 54.07, + "memory_bytes": 165989581 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.029, + "concurrency": 16, + "successful_requests": 18431, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 613.766, + "successful_latency_ms": { + "p50": 19.791, + "p95": 61.005, + "p99": 77.388 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 81.41, + "memory_bytes": 30607933 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 101.44, + "memory_bytes": 43956306 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 20.02, + "memory_bytes": 17291018 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 47.76, + "memory_bytes": 173015040 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.18, + "memory_bytes": 31195136 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 81.46, + "memory_bytes": 31719424 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 19.94, + "memory_bytes": 17343447 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 56.1, + "memory_bytes": 173119898 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 75.57, + "memory_bytes": 31132221 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 103.21, + "memory_bytes": 31824282 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 20.65, + "memory_bytes": 17427333 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 52.26, + "memory_bytes": 173539328 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 82.32, + "memory_bytes": 31740396 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 94.97, + "memory_bytes": 31299994 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.91, + "memory_bytes": 17437819 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 55.0, + "memory_bytes": 173329613 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 89.89, + "memory_bytes": 31174164 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 96.16, + "memory_bytes": 31593595 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 25.18, + "memory_bytes": 17196646 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 47.71, + "memory_bytes": 173958758 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 107.98, + "memory_bytes": 30995907 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 72.65, + "memory_bytes": 31006392 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 26.26, + "memory_bytes": 17175675 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 45.79, + "memory_bytes": 173958758 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 104.82, + "memory_bytes": 41701868 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 76.39, + "memory_bytes": 30974935 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.84, + "memory_bytes": 17731420 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 62.34, + "memory_bytes": 175741338 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 103.8, + "memory_bytes": 31362908 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 56.24, + "memory_bytes": 31048335 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 20.67, + "memory_bytes": 17228104 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 54.27, + "memory_bytes": 175846195 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 63.53, + "memory_bytes": 31488737 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 112.81, + "memory_bytes": 31383880 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.39, + "memory_bytes": 17458790 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 57.67, + "memory_bytes": 174797619 + } + ] + } + ] + }, + { + "phase": "growth", + "api_replicas": 4, + "per_replica_cpu_limit": "1.0", + "per_replica_memory_limit": "512M", + "total_api_cpu_limit": 4.0, + "total_api_memory_limit_mib": 2048, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.012, + "concurrency": 16, + "successful_requests": 32846, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1094.415, + "successful_latency_ms": { + "p50": 10.732, + "p95": 30.846, + "p99": 51.188 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.38, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 66.67, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 63.19, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 48.9, + "memory_bytes": 30440161 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 16.26, + "memory_bytes": 17217618 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 96.23, + "memory_bytes": 194825421 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 57.62, + "memory_bytes": 30545019 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 55.37, + "memory_bytes": 30461133 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 60.08, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 55.07, + "memory_bytes": 30691820 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.36, + "memory_bytes": 17804820 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 107.53, + "memory_bytes": 205625754 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.72, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 69.2, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 68.76, + "memory_bytes": 30429676 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 57.81, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.93, + "memory_bytes": 17678991 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 96.49, + "memory_bytes": 212756070 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 71.03, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 72.23, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 52.55, + "memory_bytes": 30450647 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 69.93, + "memory_bytes": 43673190 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.47, + "memory_bytes": 17406362 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 99.35, + "memory_bytes": 212651213 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.75, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 59.49, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 59.28, + "memory_bytes": 43610276 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 60.48, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 30.82, + "memory_bytes": 17049846 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 95.16, + "memory_bytes": 218103808 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 66.97, + "memory_bytes": 43767562 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 69.56, + "memory_bytes": 30607933 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 50.43, + "memory_bytes": 30817649 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 64.03, + "memory_bytes": 43578819 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.64, + "memory_bytes": 17070817 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 101.11, + "memory_bytes": 218523238 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 41.87, + "memory_bytes": 30723277 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 62.34, + "memory_bytes": 30849106 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 62.69, + "memory_bytes": 30807163 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 40.17, + "memory_bytes": 30859592 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.7, + "memory_bytes": 17647534 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 108.47, + "memory_bytes": 232993587 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 52.79, + "memory_bytes": 30712791 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 53.63, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 55.4, + "memory_bytes": 30660362 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 55.73, + "memory_bytes": 40537948 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 40.37, + "memory_bytes": 17301504 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.42, + "memory_bytes": 216111514 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 71.04, + "memory_bytes": 31226593 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 71.0, + "memory_bytes": 30639391 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 72.37, + "memory_bytes": 30691820 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 52.3, + "memory_bytes": 30953964 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.05, + "memory_bytes": 17154703 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 100.06, + "memory_bytes": 216740659 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.013, + "concurrency": 16, + "successful_requests": 32672, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1088.609, + "successful_latency_ms": { + "p50": 11.145, + "p95": 30.074, + "p99": 53.423 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 63.93, + "memory_bytes": 30209475 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 62.59, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 64.58, + "memory_bytes": 30513562 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 57.17, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 31.39, + "memory_bytes": 17332961 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 89.67, + "memory_bytes": 210344346 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 57.4, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 56.1, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 59.17, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 65.08, + "memory_bytes": 41167094 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 40.95, + "memory_bytes": 17343447 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 93.72, + "memory_bytes": 218837811 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 67.48, + "memory_bytes": 30555505 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 55.47, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 69.81, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 48.58, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.31, + "memory_bytes": 17154703 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 95.05, + "memory_bytes": 224709837 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 50.14, + "memory_bytes": 30450647 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 67.6, + "memory_bytes": 43568333 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 51.32, + "memory_bytes": 30419190 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 56.46, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.73, + "memory_bytes": 17301504 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.16, + "memory_bytes": 224709837 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 56.87, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 56.91, + "memory_bytes": 30628905 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 61.07, + "memory_bytes": 30849106 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 55.08, + "memory_bytes": 30649876 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 32.66, + "memory_bytes": 17689477 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 106.33, + "memory_bytes": 224500122 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 59.2, + "memory_bytes": 42184212 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 69.82, + "memory_bytes": 30828134 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 60.32, + "memory_bytes": 41974497 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 66.66, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 33.38, + "memory_bytes": 17490248 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 90.39, + "memory_bytes": 225129267 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 55.76, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 57.6, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 58.39, + "memory_bytes": 30555505 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 56.12, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 32.67, + "memory_bytes": 17322476 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 106.33, + "memory_bytes": 224919552 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 66.36, + "memory_bytes": 30503076 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 67.73, + "memory_bytes": 30849106 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 68.39, + "memory_bytes": 30555505 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 59.12, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 43.62, + "memory_bytes": 17280532 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 89.59, + "memory_bytes": 225024410 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 45.79, + "memory_bytes": 30849106 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 45.61, + "memory_bytes": 30628905 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 48.46, + "memory_bytes": 30901535 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 44.99, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 32.2, + "memory_bytes": 17186161 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.64, + "memory_bytes": 230896435 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 16, + "successful_requests": 33502, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1116.177, + "successful_latency_ms": { + "p50": 11.142, + "p95": 28.815, + "p99": 49.26 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 65.99, + "memory_bytes": 30754734 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 61.27, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 49.54, + "memory_bytes": 30429676 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 35.51, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 16.33, + "memory_bytes": 16976445 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 55.98, + "memory_bytes": 195978854 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 53.73, + "memory_bytes": 30545019 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 58.81, + "memory_bytes": 40621834 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 69.41, + "memory_bytes": 43736105 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 51.72, + "memory_bytes": 30419190 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.78, + "memory_bytes": 17070817 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 93.04, + "memory_bytes": 216216371 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 53.11, + "memory_bytes": 30817649 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 50.57, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 50.18, + "memory_bytes": 30628905 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 63.83, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.98, + "memory_bytes": 17395876 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 98.03, + "memory_bytes": 222088397 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 59.09, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 54.35, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 69.74, + "memory_bytes": 42561700 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 56.45, + "memory_bytes": 30450647 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.31, + "memory_bytes": 17437819 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 106.07, + "memory_bytes": 221773824 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 69.8, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 50.49, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 49.31, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 50.68, + "memory_bytes": 30440161 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.26, + "memory_bytes": 17563648 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 98.52, + "memory_bytes": 222193254 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 55.5, + "memory_bytes": 40254833 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 69.42, + "memory_bytes": 30597448 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 69.2, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 50.97, + "memory_bytes": 34823209 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.6, + "memory_bytes": 17186161 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 107.15, + "memory_bytes": 222507827 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 82.4, + "memory_bytes": 31163679 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 54.98, + "memory_bytes": 30618419 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 51.35, + "memory_bytes": 30681334 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 73.92, + "memory_bytes": 30838620 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.29, + "memory_bytes": 17500733 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 95.75, + "memory_bytes": 222402970 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 51.63, + "memory_bytes": 30607933 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 68.28, + "memory_bytes": 30891049 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 70.39, + "memory_bytes": 30660362 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 50.9, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.79, + "memory_bytes": 17783849 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 95.09, + "memory_bytes": 222507827 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 52.87, + "memory_bytes": 40747663 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 52.8, + "memory_bytes": 30922506 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 52.17, + "memory_bytes": 30670848 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 58.56, + "memory_bytes": 31100764 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.86, + "memory_bytes": 17553162 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 99.34, + "memory_bytes": 223451546 + } + ] + } + ] + }, + { + "phase": "growth", + "api_replicas": 8, + "per_replica_cpu_limit": "1.0", + "per_replica_memory_limit": "512M", + "total_api_cpu_limit": 8.0, + "total_api_memory_limit_mib": 4096, + "warmup_seconds": 5, + "baseline_row_counts": { + "users": 156, + "chats": 52, + "messages": 445522, + "sessions": 154, + "idempotency_records": 445522 + }, + "baseline_snapshot_sha256": "ddd62de0d9ba118508d4ee382c617927f5daa81f2acf769a5b49e67ac6e3db91", + "each_repetition_restored_same_baseline": true, + "repetitions": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.033, + "concurrency": 16, + "successful_requests": 34199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1138.726, + "successful_latency_ms": { + "p50": 11.784, + "p95": 29.272, + "p99": 38.508 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.26, + "memory_bytes": 29978788 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 17.48, + "memory_bytes": 29947331 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.74, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.54, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 26.18, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.17, + "memory_bytes": 29999759 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 45.49, + "memory_bytes": 30157046 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 25.89, + "memory_bytes": 29989274 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.54, + "memory_bytes": 17259561 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.67, + "memory_bytes": 216530944 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 25.98, + "memory_bytes": 30136074 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 26.92, + "memory_bytes": 29999759 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 26.82, + "memory_bytes": 30104617 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 37.31, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 44.67, + "memory_bytes": 30083645 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.87, + "memory_bytes": 30031217 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 25.73, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 27.28, + "memory_bytes": 30062674 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.87, + "memory_bytes": 17532191 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 103.75, + "memory_bytes": 229113856 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 49.17, + "memory_bytes": 30167532 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 41.91, + "memory_bytes": 43033559 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 26.44, + "memory_bytes": 30230446 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 25.87, + "memory_bytes": 30178017 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 24.24, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 49.43, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 46.29, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 25.53, + "memory_bytes": 30062674 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 34.88, + "memory_bytes": 17291018 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 102.46, + "memory_bytes": 231735296 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.66, + "memory_bytes": 30461133 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 43.87, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.1, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 42.46, + "memory_bytes": 30230446 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 45.06, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 27.79, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 25.64, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 27.28, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.34, + "memory_bytes": 17133732 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.39, + "memory_bytes": 242116198 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 47.08, + "memory_bytes": 30167532 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 47.06, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 27.02, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 43.11, + "memory_bytes": 41796239 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 28.08, + "memory_bytes": 30356275 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 52.29, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 32.98, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 46.29, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.66, + "memory_bytes": 17039360 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.74, + "memory_bytes": 244318208 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.04, + "memory_bytes": 30429676 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 27.06, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.44, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 46.93, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 46.5, + "memory_bytes": 30356275 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 27.78, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 26.6, + "memory_bytes": 30219960 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 30.96, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.14, + "memory_bytes": 16976445 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.4, + "memory_bytes": 246625075 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 45.81, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 44.36, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.59, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 29.29, + "memory_bytes": 39625687 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 26.28, + "memory_bytes": 30324818 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.35, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 41.0, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 47.14, + "memory_bytes": 30513562 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.64, + "memory_bytes": 17028874 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 101.12, + "memory_bytes": 245366784 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 22.92, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 23.57, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 41.17, + "memory_bytes": 30324818 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 22.96, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 40.69, + "memory_bytes": 30387732 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 22.75, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 38.13, + "memory_bytes": 43557847 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 23.26, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.57, + "memory_bytes": 17060332 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 101.56, + "memory_bytes": 251763098 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.64, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 44.74, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.93, + "memory_bytes": 30419190 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 24.97, + "memory_bytes": 30356275 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 24.65, + "memory_bytes": 30398218 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.19, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 46.67, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 44.47, + "memory_bytes": 30398218 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 35.02, + "memory_bytes": 17081303 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.96, + "memory_bytes": 254069965 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.015, + "concurrency": 16, + "successful_requests": 35418, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1180.016, + "successful_latency_ms": { + "p50": 11.711, + "p95": 26.361, + "p99": 36.002 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 37.61, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 43.02, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 16.44, + "memory_bytes": 29957816 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 23.71, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 16.99, + "memory_bytes": 29978788 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.42, + "memory_bytes": 30031217 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 24.85, + "memory_bytes": 29905388 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 34.59, + "memory_bytes": 42939187 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 23.9, + "memory_bytes": 16997417 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 85.42, + "memory_bytes": 208561766 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.6, + "memory_bytes": 30188503 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 26.36, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 45.35, + "memory_bytes": 30209475 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 39.79, + "memory_bytes": 30083645 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 46.87, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.84, + "memory_bytes": 30440161 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 26.85, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 27.26, + "memory_bytes": 30136074 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.14, + "memory_bytes": 17049846 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.15, + "memory_bytes": 223451546 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.14, + "memory_bytes": 30723277 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 28.63, + "memory_bytes": 39678116 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 26.04, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 45.48, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 26.88, + "memory_bytes": 30052188 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 27.04, + "memory_bytes": 30219960 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 46.49, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 47.68, + "memory_bytes": 30094131 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.56, + "memory_bytes": 17353933 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.3, + "memory_bytes": 236558746 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 48.03, + "memory_bytes": 30524047 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 44.64, + "memory_bytes": 43683676 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 36.87, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 27.69, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 44.86, + "memory_bytes": 30073160 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.1, + "memory_bytes": 30219960 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 27.02, + "memory_bytes": 30167532 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 27.79, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.82, + "memory_bytes": 17133732 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 103.85, + "memory_bytes": 239914189 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 28.1, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 27.09, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.75, + "memory_bytes": 30282875 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 45.72, + "memory_bytes": 30356275 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 46.11, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 30.26, + "memory_bytes": 40013660 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 28.11, + "memory_bytes": 34812723 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 28.08, + "memory_bytes": 30167532 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.82, + "memory_bytes": 17469276 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 103.54, + "memory_bytes": 240753050 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 44.81, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 47.74, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 27.41, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 26.21, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 27.4, + "memory_bytes": 30219960 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 41.12, + "memory_bytes": 30786191 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 27.32, + "memory_bytes": 30471619 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 26.8, + "memory_bytes": 30482104 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 40.47, + "memory_bytes": 17081303 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.18, + "memory_bytes": 244737638 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.41, + "memory_bytes": 30261903 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 26.86, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 46.27, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 44.83, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 43.85, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 44.32, + "memory_bytes": 43463475 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 32.07, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 27.36, + "memory_bytes": 30419190 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.97, + "memory_bytes": 17154703 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 102.3, + "memory_bytes": 248093082 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 40.59, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 45.63, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.13, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 28.7, + "memory_bytes": 39877345 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 27.62, + "memory_bytes": 30534533 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.64, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 45.38, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 45.94, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.36, + "memory_bytes": 17668506 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.95, + "memory_bytes": 250399949 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.61, + "memory_bytes": 30230446 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 27.67, + "memory_bytes": 30356275 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 45.95, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 45.26, + "memory_bytes": 30607933 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 50.8, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 31.1, + "memory_bytes": 40370176 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 37.04, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 26.59, + "memory_bytes": 30136074 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 40.39, + "memory_bytes": 17144218 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 102.56, + "memory_bytes": 252497101 + } + ] + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.012, + "concurrency": 16, + "successful_requests": 34848, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1161.122, + "successful_latency_ms": { + "p50": 11.745, + "p95": 28.074, + "p99": 36.688 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "resource_samples": [ + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 25.43, + "memory_bytes": 29999759 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 46.49, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 25.49, + "memory_bytes": 30167532 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 39.27, + "memory_bytes": 30094131 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 37.81, + "memory_bytes": 41680896 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 25.63, + "memory_bytes": 30607933 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 43.17, + "memory_bytes": 29968302 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 42.81, + "memory_bytes": 29999759 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 34.94, + "memory_bytes": 17039360 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 104.41, + "memory_bytes": 213385216 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.54, + "memory_bytes": 30094131 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.53, + "memory_bytes": 30157046 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.23, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 45.27, + "memory_bytes": 43421532 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 27.09, + "memory_bytes": 30178017 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 43.39, + "memory_bytes": 30083645 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 26.2, + "memory_bytes": 30020731 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 43.69, + "memory_bytes": 30115103 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.89, + "memory_bytes": 17343447 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.14, + "memory_bytes": 226702131 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 26.92, + "memory_bytes": 30146560 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 45.93, + "memory_bytes": 30198989 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 26.07, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 26.17, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 46.62, + "memory_bytes": 30230446 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 38.28, + "memory_bytes": 42593157 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 24.67, + "memory_bytes": 30062674 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 48.54, + "memory_bytes": 30408704 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 34.32, + "memory_bytes": 17112760 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 101.1, + "memory_bytes": 239494758 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.92, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 28.89, + "memory_bytes": 30240932 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 43.83, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 46.71, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 27.33, + "memory_bytes": 30450647 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 27.7, + "memory_bytes": 30125588 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 47.96, + "memory_bytes": 30492590 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 28.15, + "memory_bytes": 30450647 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.99, + "memory_bytes": 17238589 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.96, + "memory_bytes": 244632781 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 41.07, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.32, + "memory_bytes": 30230446 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 41.0, + "memory_bytes": 43631247 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 27.2, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 38.51, + "memory_bytes": 42771415 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.87, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 25.99, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 25.78, + "memory_bytes": 30251418 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 37.97, + "memory_bytes": 17112760 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.45, + "memory_bytes": 259312845 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 27.04, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 47.15, + "memory_bytes": 43421532 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 24.94, + "memory_bytes": 30303846 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 46.86, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 26.04, + "memory_bytes": 30408704 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.37, + "memory_bytes": 30314332 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 50.2, + "memory_bytes": 30576476 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 41.02, + "memory_bytes": 43316675 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 38.18, + "memory_bytes": 17395876 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.64, + "memory_bytes": 259417702 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 42.29, + "memory_bytes": 42026926 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 25.95, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.27, + "memory_bytes": 30324818 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 34.33, + "memory_bytes": 40642806 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 27.19, + "memory_bytes": 30932992 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 45.13, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 25.67, + "memory_bytes": 30565990 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 25.93, + "memory_bytes": 30272389 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.63, + "memory_bytes": 17112760 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 103.63, + "memory_bytes": 259522560 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 46.42, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 44.64, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 44.33, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 26.46, + "memory_bytes": 30345789 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 46.7, + "memory_bytes": 30691820 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 26.13, + "memory_bytes": 30387732 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 38.66, + "memory_bytes": 30366761 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 44.76, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 36.87, + "memory_bytes": 17427333 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 101.27, + "memory_bytes": 262039142 + }, + { + "container": "mtusi2026lab3-api-11", + "service": "api", + "cpu_percent": 39.97, + "memory_bytes": 30335304 + }, + { + "container": "mtusi2026lab3-api-12", + "service": "api", + "cpu_percent": 30.14, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-api-13", + "service": "api", + "cpu_percent": 46.49, + "memory_bytes": 30377247 + }, + { + "container": "mtusi2026lab3-api-14", + "service": "api", + "cpu_percent": 32.56, + "memory_bytes": 40223375 + }, + { + "container": "mtusi2026lab3-api-15", + "service": "api", + "cpu_percent": 29.91, + "memory_bytes": 30429676 + }, + { + "container": "mtusi2026lab3-api-16", + "service": "api", + "cpu_percent": 43.94, + "memory_bytes": 30387732 + }, + { + "container": "mtusi2026lab3-api-17", + "service": "api", + "cpu_percent": 28.25, + "memory_bytes": 30398218 + }, + { + "container": "mtusi2026lab3-api-18", + "service": "api", + "cpu_percent": 26.14, + "memory_bytes": 30293361 + }, + { + "container": "mtusi2026lab3-load_balancer-1", + "service": "load_balancer", + "cpu_percent": 39.77, + "memory_bytes": 17416847 + }, + { + "container": "mtusi2026lab3-postgres-1", + "service": "postgres", + "cpu_percent": 105.88, + "memory_bytes": 264870298 + } + ] + } + ] + } + ] +} diff --git a/evidence/track1/throughput.svg b/evidence/track1/throughput.svg new file mode 100644 index 00000000..726cdb5d --- /dev/null +++ b/evidence/track1/throughput.svg @@ -0,0 +1,26 @@ + + +API scaling: average successful RPS + + +212.5 +fixed / 1 + +212.7 +fixed / 2 + +156.3 +fixed / 4 + +211.7 +growth / 1 + +605.0 +growth / 2 + +1099.7 +growth / 4 + +1160.0 +growth / 8 + diff --git a/haproxy.cfg b/haproxy.cfg new file mode 100644 index 00000000..9dedf30a --- /dev/null +++ b/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + option http-keep-alive + timeout connect 2s + timeout client 15s + timeout server 15s + timeout http-request 5s + timeout check 1s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + +frontend messenger + bind :8080 + default_backend api_replicas + +backend api_replicas + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + option redispatch + default-server inter 1s fall 2 rise 2 + server-template api 1-10 api:8080 check resolvers docker resolve-prefer ipv4 init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..f5431a5f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..aae5c1ae --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + response jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, chat_id, idempotency_key), + CONSTRAINT idempotency_method_post CHECK (method = 'POST'), + CONSTRAINT idempotency_hash_sha256 CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX idempotency_records_expiry_idx ON idempotency_records (expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..177a9775 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100755 index 00000000..811b13c7 --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +SECRETS_DIR="$ROOT/secrets" +PASSWORD_FILE="$SECRETS_DIR/postgres_password" +DATABASE_URL_FILE="$SECRETS_DIR/database_url" + +if [ -e "$PASSWORD_FILE" ] || [ -e "$DATABASE_URL_FILE" ]; then + printf '%s\n' 'Secret files already exist; refusing to overwrite them.' >&2 + exit 1 +fi + +command -v openssl >/dev/null 2>&1 || { + printf '%s\n' 'openssl is required to generate local credentials.' >&2 + exit 1 +} + +mkdir -p "$SECRETS_DIR" +chmod 700 "$SECRETS_DIR" +umask 077 +password=$(openssl rand -hex 32) +printf '%s' "$password" > "$PASSWORD_FILE" +printf 'postgresql://messenger:%s@postgres:5432/messenger' "$password" > "$DATABASE_URL_FILE" +chmod 644 "$PASSWORD_FILE" "$DATABASE_URL_FILE" +unset password +printf '%s\n' 'Local PostgreSQL secret files created.' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..08e2218d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +docker compose --env-file .env restart postgres +docker compose --env-file .env up -d --wait --wait-timeout 60 postgres +docker compose --env-file .env restart api load_balancer diff --git a/scripts/rolling-restart.sh b/scripts/rolling-restart.sh new file mode 100755 index 00000000..3d550388 --- /dev/null +++ b/scripts/rolling-restart.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 60 +containers=$(docker compose --env-file .env ps -q api) +if [ -z "$containers" ]; then + printf '%s\n' 'No running API containers found.' >&2 + exit 1 +fi + +for container in $containers; do + docker stop --time 15 "$container" >/dev/null + docker start "$container" >/dev/null + attempts=0 + while [ "$attempts" -lt 40 ]; do + state=$(docker inspect --format '{{.State.Health.Status}}' "$container") + if [ "$state" = healthy ]; then + printf 'PASS: API container %s restarted and is healthy\n' "$container" + break + fi + if [ "$state" = unhealthy ] || [ "$state" = none ]; then + printf 'API container %s failed health check (state: %s)\n' "$container" "$state" >&2 + exit 1 + fi + attempts=$((attempts + 1)) + sleep 1 + done + if [ "$attempts" -eq 40 ]; then + printf 'API container %s did not become healthy in 40 seconds\n' "$container" >&2 + exit 1 + fi +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..55180627 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,32 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +target_container= +for container in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + target_container=$container + break + fi +done + +if [ -z "$target_container" ]; then + printf '%s\n' 'No running API container matches TARGET_INSTANCE_ID.' >&2 + exit 1 +fi + +if [ -n "${FAILOVER_EVIDENCE_FILE:-}" ]; then + case "$FAILOVER_EVIDENCE_FILE" in + /*) evidence_file=$FAILOVER_EVIDENCE_FILE ;; + *) evidence_file="$ROOT/$FAILOVER_EVIDENCE_FILE" ;; + esac + printf '%s target_instance=%s container=%s\n' \ + "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$TARGET_INSTANCE_ID" "$target_container" \ + >> "$evidence_file" +fi + +docker stop --time 2 "$target_container" >/dev/null diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..5bc8da31 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..d5229ba9 --- /dev/null +++ b/src/app.py @@ -0,0 +1,39 @@ +import logging +import os +import signal +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .store import Store + +LOGGER = logging.getLogger("messenger") + + +def run_server(): + store = Store() + store.open() + try: + handler = partial(MessengerHandler, store=store) + server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + run_server() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..419b28ad --- /dev/null +++ b/src/config.py @@ -0,0 +1,29 @@ +import os +import re +import socket +import uuid + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", socket.gethostname() or uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL_SECONDS = int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if IDEMPOTENCY_TTL_SECONDS < 86400: + raise RuntimeError("IDEMPOTENCY_TTL_SECONDS must be at least 86400") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..35cffe66 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,18 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass + + +class IdempotencyConflictError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 00000000..8a3adffb --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,350 @@ +import base64 +import json +import logging +import threading +import uuid +from http.server import BaseHTTPRequestHandler +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit + +import psycopg +from psycopg_pool import PoolTimeout + +from .config import ( + IDEMPOTENCY_KEY_RE, + INSTANCE_ID, + MAX_INFLIGHT_REQUESTS, + MAX_REQUEST_BODY_BYTES, + USER_RE, +) +from .errors import ApiError, CacheUnavailableError, IdempotencyConflictError +from .security import hash_password, isoformat, valid_uuid +from .store import Store + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError): + raise ApiError(503, "unavailable", "Database is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except IdempotencyConflictError: + self._send_error(409, "conflict", "Idempotency-Key was already used with another request", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat_id = str(uuid.UUID(chat_id)) + idempotency_key = self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"text"}: + raise ApiError(400, "invalid_request", "Expected text") + text = body["text"] + if not isinstance(text, str) or not text.strip() or len(text) > 4000: + raise ApiError(400, "invalid_request", "Invalid message text") + canonical_body = json.dumps( + {"text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + message = self.store.create_idempotent_message( + chat_id, + actor["id"], + text, + idempotency_key, + canonical_body, + ) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1ae27009 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6ec821f7 --- /dev/null +++ b/src/security.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import secrets +import uuid +from datetime import datetime, timezone + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + salt_text = base64.urlsafe_b64encode(salt).decode("ascii") + hash_text = base64.urlsafe_b64encode(derived).decode("ascii") + return f"scrypt$32768$8$1${salt_text}${hash_text}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError, UnicodeError): + return False diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..60fd3d56 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..476dba13 --- /dev/null +++ b/src/store.py @@ -0,0 +1,394 @@ +import base64 +import hashlib +import hmac +import json +import logging +import math +import os +import secrets +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import redis +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + IDEMPOTENCY_TTL_SECONDS, + SESSION_TTL_SECONDS, +) +from .errors import CacheUnavailableError, IdempotencyConflictError +from .security import isoformat, utc_now, valid_uuid, verify_password + +LOGGER = logging.getLogger("messenger.store") + + +def database_url() -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + file_path = os.getenv("DATABASE_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE") + + +class Store: + """PostgreSQL source of truth and optional Redis session lookup cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + if not row or not verify_password(password, row["password_hash"]): + return None + return self.user(row) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_idempotent_message( + self, + chat_id: str, + sender_id: str, + text: str, + idempotency_key: str, + canonical_body: str, + ) -> Optional[Dict[str, Any]]: + request_hash = hashlib.sha256(canonical_body.encode("utf-8")).digest() + lock_material = "\0".join((sender_id, "POST", chat_id, idempotency_key)).encode("utf-8") + lock_key = int.from_bytes(hashlib.sha256(lock_material).digest()[:8], "big", signed=True) + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = conn.execute( + "SELECT request_hash, response, expires_at " + "FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s " + "FOR UPDATE", + (sender_id, chat_id, idempotency_key), + ).fetchone() + if existing and existing["expires_at"] > utc_now(): + if not hmac.compare_digest(bytes(existing["request_hash"]), request_hash): + raise IdempotencyConflictError + return existing["response"] + if existing: + conn.execute( + "DELETE FROM idempotency_records " + "WHERE user_id = %s AND method = 'POST' AND chat_id = %s AND idempotency_key = %s", + (sender_id, chat_id, idempotency_key), + ) + + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + message = self.message(row) + conn.execute( + "INSERT INTO idempotency_records " + "(user_id, method, chat_id, idempotency_key, request_hash, response, expires_at) " + "VALUES (%s, 'POST', %s, %s, %s, %s::jsonb, %s)", + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + json.dumps(message, ensure_ascii=False, separators=(",", ":")), + utc_now() + timedelta(seconds=IDEMPOTENCY_TTL_SECONDS), + ), + ) + return message + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row) for row in page], next_cursor diff --git a/student-tests/test_idempotency_race.py b/student-tests/test_idempotency_race.py new file mode 100644 index 00000000..646869a7 --- /dev/null +++ b/student-tests/test_idempotency_race.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Prove a concurrent idempotency-key race through the load balancer.""" +from concurrent.futures import ThreadPoolExecutor +import threading +import sys +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency integration check") + client = Client(args.base_url, args.ca_file) + alice, _, _, chat = client.bootstrap() + key = "race-" + uuid.uuid4().hex + text = "concurrent-" + uuid.uuid4().hex + count = 12 + barrier = threading.Barrier(count) + + def send_once(_): + worker = Client(args.base_url, args.ca_file) + barrier.wait(timeout=10) + return worker.message(alice, chat, text, key=key) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send_once, range(count))) + + instance_ids = {response["headers"]["X-Instance-Id"] for response in responses} + messages = [response["json"] for response in responses] + check(len(instance_ids) >= 2, "concurrent requests did not reach multiple API replicas") + check(all(message == messages[0] for message in messages), "same-key requests returned different messages") + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=alice, + schema="MessagePage", + )["json"] + matches = [message for message in page["items"] if message["text"] == text] + check(len(matches) == 1 and matches[0] == messages[0], "concurrent retry persisted duplicate messages") + print(f"PASS: {count} concurrent same-key requests on {len(instance_ids)} replicas returned one message") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1_benchmark.py b/student-tests/track1_benchmark.py new file mode 100644 index 00000000..fe779578 --- /dev/null +++ b/student-tests/track1_benchmark.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +"""Repeatable 1/2/4 replica throughput experiment with Docker resource samples.""" +import argparse +import json +import math +import platform +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import time +import hashlib + +ROOT = Path(__file__).resolve().parents[1] +LOAD_TEST = ROOT / "tests" / "load.py" +OUTPUT_DIR = ROOT / "evidence" / "track1" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def run(command, *, env=None, capture=True): + return subprocess.run( + command, + cwd=ROOT, + env=env, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + + +def compose_environment(cpu_limit, memory_limit): + import os + + env = os.environ.copy() + env["API_CPU_LIMIT"] = cpu_limit + env["API_MEMORY_LIMIT"] = memory_limit + return env + + +def database_counts(): + query = ( + "SELECT json_build_object(" + "'users', (SELECT count(*) FROM users), " + "'chats', (SELECT count(*) FROM chats), " + "'messages', (SELECT count(*) FROM messages), " + "'sessions', (SELECT count(*) FROM sessions), " + "'idempotency_records', (SELECT count(*) FROM idempotency_records)" + ")::text" + ) + result = run( + COMPOSE + + [ + "exec", + "-T", + "postgres", + "psql", + "-X", + "-qAt", + "-U", + "messenger", + "-d", + "messenger", + "-c", + query, + ] + ) + return json.loads(result.stdout) + + +def create_database_snapshot(destination): + with destination.open("wb") as dump: + result = subprocess.run( + COMPOSE + ["exec", "-T", "postgres", "pg_dump", "-Fc", "-U", "messenger", "-d", "messenger"], + cwd=ROOT, + check=False, + stdout=dump, + stderr=subprocess.PIPE, + ) + if result.returncode: + raise RuntimeError("failed to create PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace")) + digest_builder = hashlib.sha256() + with destination.open("rb") as dump: + for chunk in iter(lambda: dump.read(1024 * 1024), b""): + digest_builder.update(chunk) + digest = digest_builder.hexdigest() + return digest, database_counts() + + +def restore_database_snapshot(snapshot, baseline_counts, api_replicas, cpu_limit, memory_limit): + env = compose_environment(cpu_limit, memory_limit) + run(COMPOSE + ["stop", "api"]) + with snapshot.open("rb") as dump: + result = subprocess.run( + COMPOSE + + [ + "exec", + "-T", + "postgres", + "pg_restore", + "-U", + "messenger", + "--clean", + "--if-exists", + "--no-owner", + "--no-acl", + "--exit-on-error", + "--dbname=messenger", + ], + cwd=ROOT, + check=False, + stdin=dump, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if result.returncode: + raise RuntimeError("failed to restore PostgreSQL baseline snapshot: " + result.stderr.decode(errors="replace")) + restored_counts = database_counts() + if restored_counts != baseline_counts: + raise RuntimeError( + "restored PostgreSQL baseline has different row counts: " + + json.dumps(restored_counts, sort_keys=True) + ) + run( + COMPOSE + + [ + "up", + "-d", + "--no-deps", + "--build", + "--force-recreate", + "--scale", + f"api={api_replicas}", + "--wait", + "--wait-timeout", + "120", + "api", + ], + env=env, + ) + + +def docker_stats(): + containers = run( + ["docker", "compose", "--env-file", ".env", "ps", "-q"], + ).stdout.splitlines() + if not containers: + return [] + result = run( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}", + *containers, + ] + ) + samples = [] + for line in result.stdout.splitlines(): + name, cpu, memory = line.split("|", 2) + used = memory.split("/", 1)[0].strip() + samples.append( + { + "container": name, + "service": "api" if "-api-" in name else "postgres" if "-postgres-" in name else "load_balancer", + "cpu_percent": float(cpu.rstrip("%")), + "memory_bytes": parse_bytes(used), + } + ) + return samples + + +def parse_bytes(value): + match = re.fullmatch(r"([0-9.]+)\s*([KMG]i?B|B)", value) + if not match: + return None + amount = float(match.group(1)) + unit = match.group(2) + power = {"B": 0, "KB": 1, "KiB": 1, "MB": 2, "MiB": 2, "GB": 3, "GiB": 3}[unit] + return round(amount * 1024**power) + + +def run_load(name, duration, concurrency, repetition, base_url): + output = OUTPUT_DIR / f"{name}-run-{repetition}.json" + command = [ + sys.executable, + str(LOAD_TEST), + "--base-url", + base_url, + "--duration", + str(duration), + "--concurrency", + str(concurrency), + "--output", + str(output.relative_to(ROOT)), + ] + process = subprocess.Popen( + command, + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + samples = [] + while process.poll() is None: + samples.extend(docker_stats()) + time.sleep(1) + stdout, stderr = process.communicate() + if process.returncode: + raise RuntimeError(f"load run failed:\n{stdout}\n{stderr}") + metrics = json.loads(output.read_text(encoding="utf-8")) + metrics["resource_samples"] = samples + return metrics + + +def system_info(): + def command_output(command): + try: + return run(command).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "unavailable" + + return { + "platform": platform.platform(), + "processor": platform.processor() or "unavailable", + "cpu_count": __import__("os").cpu_count(), + "host_memory_bytes": command_output(["sysctl", "-n", "hw.memsize"]), + "docker_server": command_output(["docker", "version", "--format", "{{.Server.Version}}"]), + "docker_compose": command_output(["docker", "compose", "version", "--short"]), + } + + +def configure_limits(count, phase): + if phase == "fixed": + return f"{1 / count:.4f}", f"{512 // count}M" + return "1.0", "512M" + + +def experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts): + cpu_limit, memory_limit = configure_limits(count, phase) + restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit) + run_load(f"{phase}-{count}-warmup", args.warmup, args.concurrency, 0, args.base_url) + warmup_counts = database_counts() + if warmup_counts == baseline_counts: + raise RuntimeError("warm-up did not create synthetic workload data") + + repetitions = [] + for repetition in range(1, args.repetitions + 1): + restore_database_snapshot(snapshot, baseline_counts, count, cpu_limit, memory_limit) + repetitions.append(run_load(f"{phase}-{count}", args.duration, args.concurrency, repetition, args.base_url)) + if database_counts() == baseline_counts: + raise RuntimeError(f"{phase}-{count} repetition {repetition} did not write benchmark data") + return { + "phase": phase, + "api_replicas": count, + "per_replica_cpu_limit": cpu_limit, + "per_replica_memory_limit": memory_limit, + "total_api_cpu_limit": round(float(cpu_limit) * count, 4), + "total_api_memory_limit_mib": int(memory_limit[:-1]) * count, + "warmup_seconds": args.warmup, + "baseline_row_counts": baseline_counts, + "baseline_snapshot_sha256": snapshot_hash, + "each_repetition_restored_same_baseline": True, + "repetitions": repetitions, + } + + +def make_graph(cases): + width, height = 900, 420 + margin = 60 + values = [ + ( + f"{case['phase']} / {case['api_replicas']}", + sum(run["successful_rps"] for run in case["repetitions"]) / len(case["repetitions"]), + "#3572A5" if case["phase"] == "fixed" else "#E38B29", + ) + for case in cases + ] + top = max((value for _, value, _ in values), default=1) or 1 + graph_height = height - 2 * margin + bar_width = 78 + gap = (width - 2 * margin - bar_width * len(values)) / max(1, len(values) - 1) + parts = [ + f'', + '', + 'API scaling: average successful RPS', + f'', + ] + for index, (label, value, color) in enumerate(values): + x = margin + index * (bar_width + gap) + bar_height = value / top * graph_height + y = height - margin - bar_height + parts.append(f'') + parts.append(f'{value:.1f}') + parts.append(f'{label}') + parts.append("") + (OUTPUT_DIR / "throughput.svg").write_text("\n".join(parts) + "\n", encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--duration", type=int, default=30) + parser.add_argument("--warmup", type=int, default=5) + parser.add_argument("--concurrency", type=int, default=16) + parser.add_argument("--repetitions", type=int, default=3) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + args = parser.parse_args() + if not 1 <= args.duration <= 3600 or not 1 <= args.warmup <= 600: + parser.error("duration and warmup must be positive and bounded") + if not 1 <= args.concurrency <= 128 or not 1 <= args.repetitions <= 10: + parser.error("concurrency or repetitions out of range") + + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + results_file = OUTPUT_DIR / "results.json" + + with tempfile.TemporaryDirectory(prefix="lab3-track1-") as temporary_directory: + snapshot = Path(temporary_directory) / "baseline.dump" + run(COMPOSE + ["stop", "api"]) + baseline_counts = None + try: + snapshot_hash, baseline_counts = create_database_snapshot(snapshot) + cases = [] + for phase, replica_counts in (("fixed", (1, 2, 4)), ("growth", (1, 2, 4, 8))): + for count in replica_counts: + cases.append( + experiment_case(count, phase, args, snapshot, snapshot_hash, baseline_counts) + ) + summary = { + "method": ( + "closed-loop POST message workload via HAProxy; same concurrency and per-run " + "restored PostgreSQL custom-format baseline snapshot" + ), + "database_baseline": { + "snapshot_sha256": snapshot_hash, + "row_counts": baseline_counts, + "restore_before_each_warmup_and_measured_repetition": True, + "state_after_experiment": "baseline restored; all synthetic benchmark writes removed", + }, + "system": system_info(), + "database_and_lb_limits": {"postgres": "1 CPU / 1 GiB", "HAProxy": "0.5 CPU / 128 MiB"}, + "client_concurrency": args.concurrency, + "requested_duration_seconds": args.duration, + "repetitions_per_case": args.repetitions, + "cases": cases, + } + results_file.write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + make_graph(cases) + for case in cases: + values = [run["successful_rps"] for run in case["repetitions"]] + deviation = math.sqrt(sum((value - sum(values) / len(values)) ** 2 for value in values) / len(values)) + print( + f"{case['phase']:6} replicas={case['api_replicas']} " + f"mean_rps={sum(values) / len(values):.2f} stdev={deviation:.2f}" + ) + print(f"Evidence saved under {OUTPUT_DIR.relative_to(ROOT)}") + finally: + if baseline_counts is not None: + restore_database_snapshot(snapshot, baseline_counts, 2, "1.0", "512M") + else: + run( + COMPOSE + + [ + "up", + "-d", + "--no-deps", + "--build", + "--force-recreate", + "--scale", + "api=2", + "--wait", + "--wait-timeout", + "120", + "api", + ], + env=compose_environment("1.0", "512M"), + ) + + +if __name__ == "__main__": + main()