diff --git a/experiments/README.md b/experiments/README.md new file mode 100644 index 00000000..1597b859 --- /dev/null +++ b/experiments/README.md @@ -0,0 +1,23 @@ +# Эксперимент: proxy upload и presigned POST + +Сравниваем трафик API, память и время загрузки. Результаты всех серий — в корневом [evidence/upload-comparison](../evidence/upload-comparison/). + +## Документация + +- [Гипотеза и критерии](docs/HYPOTHESIS.md) +- [Методика и метрики](docs/METHODOLOGY.md) +- [Ограничения](docs/LIMITATIONS.md) +- [Запуск, графики и повтор](docs/RUNBOOK.md) + +## Структура + +| Каталог / файл | Назначение | +| --- | --- | +| [config/protocol.json](config/protocol.json) | Параметры и пороги эксперимента | +| [config/compose.yml](config/compose.yml) | Отдельный стенд | +| [src](src/) | Генератор данных, нагрузка и измерители | +| [tests](tests/) | Проверки инструментов | +| [plot.py](plot.py) | Графики из результатов серии | +| [setup.sh](setup.sh) | Создание корневого `.venv` для анализа | + +Начни с [инструкции запуска](docs/RUNBOOK.md): проверка окружения → пилот → основная серия → графики. Пилот проверяет инструменты и не подтверждает гипотезу. Измерения пока не выполнены. diff --git a/experiments/config/compose.yml b/experiments/config/compose.yml new file mode 100644 index 00000000..922d9e52 --- /dev/null +++ b/experiments/config/compose.yml @@ -0,0 +1,157 @@ +services: + db: + image: ${DB_IMAGE} + pull_policy: never + cpus: 1 + mem_limit: 512m + memswap_limit: 512m + networks: [data] + environment: + POSTGRES_DB: messenger_exp + POSTGRES_USER: messenger_exp + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + volumes: + - postgres_data:/var/lib/postgresql/data + - ${SECRET_DIR}/db_password:/run/secrets/db_password:ro + healthcheck: + test: [CMD-SHELL, pg_isready -U messenger_exp -d messenger_exp] + interval: 2s + timeout: 2s + retries: 30 + redis: + image: ${REDIS_IMAGE} + pull_policy: never + cpus: 0.25 + mem_limit: 128m + memswap_limit: 128m + networks: [data] + command: [redis-server, --appendonly, "yes", --appendfsync, everysec, --dir, /data] + volumes: [redis_data:/data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + s3: + image: ${S3_IMAGE} + pull_policy: never + cpus: 1 + mem_limit: 512m + memswap_limit: 512m + networks: [edge, data] + ports: ["127.0.0.1:${S3_PORT}:9000"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/root_access + MINIO_ROOT_PASSWORD_FILE: /run/secrets/root_secret + MINIO_BROWSER: "off" + MINIO_API_CORS_ALLOW_ORIGIN: http://127.0.0.1:5173,http://localhost:5173 + volumes: + - s3_data:/data + - ${SECRET_DIR}/root_access:/run/secrets/root_access:ro + - ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro + healthcheck: + test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/ready] + interval: 2s + timeout: 2s + retries: 30 + migrate: + image: ${APP_IMAGE} + pull_policy: never + cpus: 0.5 + mem_limit: 128m + networks: [data] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + volumes: + - ${SECRET_DIR}/database_url:/run/secrets/database_url:ro + command: [./messenger, migrate, up] + depends_on: + db: {condition: service_healthy} + s3-bootstrap: + image: ${APP_IMAGE} + pull_policy: never + cpus: 0.5 + mem_limit: 128m + networks: [data] + environment: + <<: &s3 + S3_ENDPOINT: http://s3:9000 + S3_PUBLIC_ENDPOINT: http://127.0.0.1:${S3_PORT} + S3_REGION: us-east-1 + S3_BUCKET: messenger-experiment + S3_ACCESS_KEY_ID_FILE: /run/secrets/api_access + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/api_secret + S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/root_access + S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/root_secret + volumes: + - ${SECRET_DIR}/api_access:/run/secrets/api_access:ro + - ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro + - ${SECRET_DIR}/root_access:/run/secrets/root_access:ro + - ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro + command: [./messenger, s3-bootstrap] + depends_on: + s3: {condition: service_healthy} + api: + image: ${APP_IMAGE} + pull_policy: never + cpus: 0.5 + mem_limit: 512m + memswap_limit: 512m + deploy: {replicas: 2} + networks: [proxy, data] + stop_grace_period: 15s + environment: + <<: *s3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: experiment + GOMAXPROCS: "1" + GOMEMLIMIT: 384MiB + DATABASE_MAX_CONNS: "10" + SESSION_TTL_SECONDS: "3600" + IDEMPOTENCY_TTL_SECONDS: "86400" + UPLOAD_TMP_DIR: /tmp/uploads + UPLOAD_CLEANUP_GRACE_SECONDS: "600" + volumes: + - ${SECRET_DIR}/api_access:/run/secrets/api_access:ro + - ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro + - ${SECRET_DIR}/database_url:/run/secrets/database_url:ro + - ${SECRET_DIR}/cache_url:/run/secrets/cache_url:ro + tmpfs: ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"] + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: [CMD, ./messenger, healthcheck] + interval: 2s + timeout: 3s + retries: 30 + haproxy: + image: ${HAPROXY_IMAGE} + pull_policy: never + cpus: 0.5 + mem_limit: 128m + memswap_limit: 128m + networks: [edge, proxy] + environment: {API_REPLICAS: "2", APP_PORT: "8080"} + ports: ["127.0.0.1:${API_PORT}:8080", "127.0.0.1:${STATS_PORT}:8404"] + volumes: + - ${REPO_ROOT}/infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 2s + timeout: 3s + retries: 30 +volumes: + postgres_data: + redis_data: + s3_data: +networks: + edge: + proxy: {internal: true} + data: {internal: true} diff --git a/experiments/config/protocol.json b/experiments/config/protocol.json new file mode 100644 index 00000000..1f89c9d2 --- /dev/null +++ b/experiments/config/protocol.json @@ -0,0 +1,53 @@ +{ + "version": 1, + "name": "proxy-versus-presigned-post", + "purpose": "confirmatory", + "hypothesis": "Presigned POST uses at most 65% of proxy API RX+TX bytes per ready, byte-verified attachment in each profile, without request errors, OOM or API restarts.", + "seed": 20261018, + "repetitions": 3, + "clients": 10, + "waves": 3, + "wave_pause_seconds": 1, + "warmup_clients": 4, + "settle_seconds": 2, + "idle_seconds": 5, + "sample_interval_seconds": 0.1, + "request_timeout_seconds": 35, + "flow_timeout_seconds": 90, + "ports": {"api": 18080, "s3": 19000, "stats": 18404}, + "resources": { + "api_replicas": 2, + "api_cpus": 0.5, + "api_memory_bytes": 536870912, + "api_tmpfs_bytes": 16777216, + "api_go_memory_limit": "384MiB", + "api_gomaxprocs": 1, + "database_max_connections_per_replica": 10, + "db_cpus": 1, + "db_memory_bytes": 536870912, + "s3_cpus": 1, + "s3_memory_bytes": 536870912, + "redis_cpus": 0.25, + "redis_memory_bytes": 134217728, + "haproxy_cpus": 0.5, + "haproxy_memory_bytes": 134217728 + }, + "images": {"db": "postgres:16.10-bookworm", "redis": "redis:7.4.11-bookworm", "haproxy": "haproxy:3.2.25-alpine"}, + "profiles": [ + {"name": "small-moderate", "size_bytes": 1048576, "width": 1024, "height": 1024}, + {"name": "large-moderate", "size_bytes": 10485760, "width": 1024, "height": 1024}, + {"name": "small-pixels40m", "size_bytes": 1048576, "width": 8000, "height": 5000}, + {"name": "large-pixels40m", "size_bytes": 10485760, "width": 8000, "height": 5000} + ], + "criteria": { + "median_direct_proxy_traffic_ratio_max": 0.65, + "each_pair_traffic_ratio_max": 0.70, + "errors_allowed": 0, + "api_peak_strictly_below_bytes": 536870912, + "start_spread_seconds_max": 0.1, + "idle_background_payload_fraction_max": 0.05, + "sampler_coverage_fraction_min": 0.7, + "sampler_gap_seconds_max": 1.0, + "generator_cpu_cores_max": 0.8 + } +} diff --git a/experiments/docs/HYPOTHESIS.md b/experiments/docs/HYPOTHESIS.md new file mode 100644 index 00000000..5a6685ea --- /dev/null +++ b/experiments/docs/HYPOTHESIS.md @@ -0,0 +1,44 @@ +# Гипотеза, версия 1 + +[README](../README.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md) + +**H1:** presigned POST уменьшает суммарный RX+TX двух API на готовое, побайтно проверенное вложение минимум на 35% по медиане парных повторов каждого профиля. Ошибок, OOM и рестартов быть не должно. + +Proxy принимает файл и передаёт его S3. Direct отправляет файл сразу в S3, но API скачивает конечный объект для проверки. Поэтому ожидается снижение трафика API; снижение RAM не обещается. + +## Показатель и критерии + +```text +b = сумма RX+TX двух API за окно / число ready-вложений +R = b_direct / b_proxy для одного профиля и повтора +``` + +Все ready-вложения затем проверяются по длине и SHA-256. Фон измеряется отдельно и не вычитается. + +Для подтверждения нужны все условия: + +- По каждому из 4 профилей есть 3 полные пары: медиана R ≤ 0.65, каждый R ≤ 0.70. +- Все 720 измеряемых загрузок и 96 загрузок прогрева успешны; оригиналы измеряемых загрузок совпали. +- Нет OOM, рестартов и завершений API/S3; lifetime peak каждой API строго ниже 512 МиБ. +- Обе API участвовали в прогреве и измерении. + +## Пригодность измерений + +| Проверка | Порог | +| --- | --- | +| Разброс старта клиентов в волне | ≤100 мс | +| Покрытие RAM-точками с шагом 100 мс | ≥70% | +| Максимальный промежуток между точками, включая границы окна | ≤1 с | +| Оценённый фон API относительно payload прогона | ≤5% | +| Средний CPU генератора | <0.8 ядра | + +Нужны полные записи запросов, проверок и счётчиков без необъяснённых сбросов. + +## Решение + +- `confirmed_in_tested_conditions` — все критерии выполнены. +- `rejected` — ошибки системы/оригиналов, OOM/рестарт/предел памяти либо превышение порогов трафика на пригодных данных. +- `inconclusive` — данных недостаточно или измерители/запуск нарушены, системного опровержения нет. +- `pilot_not_for_hypothesis` — пилот, независимо от чисел. + +Ошибка измерителя не отменяет известный OOM или отказ приложения. Отмена пользователем не считается отказом приложения. Все попытки сохраняются. Пороги фиксируются до запуска; изменение требует новой версии протокола и новой серии. diff --git a/experiments/docs/LIMITATIONS.md b/experiments/docs/LIMITATIONS.md new file mode 100644 index 00000000..cd5355e3 --- /dev/null +++ b/experiments/docs/LIMITATIONS.md @@ -0,0 +1,16 @@ +# Ограничения + +[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Запуск](RUNBOOK.md) + +- **Один WSL/Docker-host.** Сервисы и генератор делят ресурсы. Повторы не являются независимыми машинами; посторонняя нагрузка и page cache могут влиять на результат. +- **Локальный Python-клиент, без TLS.** Новое TCP-соединение на каждой стадии. Результат не переносится напрямую на WAN, браузер и React/Web Crypto. +- **Только синтетический RGB8 PNG.** Дополнительный chunk управляет encoded-размером. Это не модель JPEG-фотографий или всех допустимых изображений. +- **Короткие волны нагрузки.** 10 клиентов, максимум два декодера. Эксперимент не определяет максимальный RPS или длительную устойчивость. Три пары дают ограниченную оценку разброса; p99 из 30 запросов близок к максимуму. +- **Direct не устраняет стоимость обработки.** API читает и декодирует final; S3 делает COPY и хранит staging. Снижение трафика API не означает снижение общей стоимости системы. Дисковый I/O отдельно не исследуется. +- **RAM-пики различаются.** Sampled max может пропустить короткий всплеск; kernel lifetime peak включает startup и прогрев. tmpfs и sampler входят в память cgroup; GOMEMLIMIT не равен всей памяти контейнера. +- **Измерители создают нагрузку.** Sampler, хеширование и запись evidence расходуют CPU/RAM. Порог CPU генератора лишь сигнал возможного ограничения. Сетевые счётчики включают SQL, Redis и healthcheck; фон не вычитается. +- **Повтор зависит от артефактов.** Нужны сохранённые fixtures, исходники и image ID. Пересборка после удаления образов может дать другую базу; версии хоста, Python и zlib фиксируются. +- **Отказы не скрываются.** Все попытки сохраняются. Ошибки/OOM опровергают работоспособность; пропуски измерителя могут сделать результат неопределённым. Исправление проверяется новой серией. +- **Без fault/security-нагрузки.** ACL, подписи и компенсация проверяются отдельно. Cleanup worker исключён из экспериментального стенда, чтобы не менять измеряемый трафик. + +До фактических измерений и проверки качества данных выводов о выигрыше нет. diff --git a/experiments/docs/METHODOLOGY.md b/experiments/docs/METHODOLOGY.md new file mode 100644 index 00000000..515803bb --- /dev/null +++ b/experiments/docs/METHODOLOGY.md @@ -0,0 +1,56 @@ +# Методика, версия 1 + +[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md) + +## Сравнение + +- **Proxy:** binary POST в API до `201 ready`. +- **Direct:** SHA-256 на клиенте → init API → presigned POST в S3 → complete API до `201 ready`. + +Повторов запросов нет. Обе ветки используют один образ API и полную проверку изображения. Direct включает COPY внутри S3 и чтение конечного объекта API. + +## Стенд и данные + +| Сервис | CPU | RAM | +| --- | --- | --- | +| API ×2 | по 0.5 | по 512 МиБ | +| PostgreSQL / MinIO | по 1 | по 512 МиБ | +| Redis | 0.25 | 128 МиБ | +| HAProxy | 0.5 | 128 МиБ | + +Каждая API: один декодер, tmpfs 16 МиБ, GOMAXPROCS=1, GOMEMLIMIT=384MiB, пул БД=10. Swap отключён. Таймауты: операция API 25 с, HTTP 30 с, HAProxy/socket 35 с; бюджет flow 90 с проверяется между стадиями. + +Четыре RGB8 PNG-профиля: **1 и 10 МиБ × 1024×1024 и 8000×5000 пикселей**. Размер доводится корректным ancillary chunk `npAD` перед IEND; в паре размеров одной геометрии пиксели одинаковые. Файлы и SHA-256 сохраняются. + +## Порядок + +Seed `20261018`, 3 парных повтора каждого профиля, порядок пар перемешан; 6 пар proxy→direct и 6 direct→proxy. Всего 24 прогона, 720 измеряемых загрузок. + +Для каждого прогона: + +1. Поднять отдельный Compose-проект со свежими volumes и закреплёнными image ID. Создать двух пользователей и чат. +2. Выполнить 4 параллельных загрузки прогрева, проверить метаданные; подождать 2 с. +3. Запустить sampler API/S3 с шагом 100 мс. Измерить фон за 5 с. +4. Снять начальные счётчики. Выполнить 3 волны по 10 клиентов со стартовым барьером и паузой 1 с между волнами. +5. Снять конечные счётчики и остановить sampler. Затем скачать оригиналы вторым участником и проверить длину, MIME и SHA-256. +6. Сохранить состояния и ошибки, остановить только этот проект. По умолчанию удалить его volumes. + +Файл заранее находится в RAM клиента; direct checksum входит во время flow. Исходники не меняются, параллельная нагрузка запрещена. Ошибки запросов сохраняются, серия продолжается; отмена, изменение исходников или сбой окружения/уборки останавливают запуск. + +## Метрики и сохранение + +| Метрика | Источник / смысл | +| --- | --- | +| API RX+TX | Разность `/proc/net/dev` всех интерфейсов кроме lo, сумма двух API; служебный обмен включён | +| S3 RX+TX | Отдельный показатель, не складывается с API | +| RAM в окне | Максимум sampled `memory.current`, baseline и интервалы точек | +| Lifetime RAM | `memory.peak` до/после, включая startup и прогрев | +| CPU / throttling / OOM | `cpu.stat`, `memory.events`, состояния и рестарты контейнеров | +| Время / goodput | Monotonic clock; стадии, p50/p95/p99 успешных flow, ready/сек за окно | +| Генератор | CPU, RSS, фактический разброс старта клиентов | + +`prepare` фиксирует протокол, план, исходники, fixtures и image ID в `seal.json`. Перед прогонами проверяются контрольные суммы. `summarize` создаёт `metrics.json`, `metrics.csv` и provenance; сравнивает парные R и их медиану отдельно по профилям. + +В корневом `evidence/upload-comparison/<серия>/`: замороженная конфигурация и исходники, `cases/*` с запросами, RAM-точками, счётчиками, проверками файлов и ошибками. Секреты и подписанные формы не записываются. Временные credentials находятся вне evidence в `/tmp/highload-upload-exp-*`. + +Источники измерителей: [сетевая статистика Linux](https://www.kernel.org/doc/html/latest/networking/statistics.html), [cgroup v2](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html). diff --git a/experiments/docs/RUNBOOK.md b/experiments/docs/RUNBOOK.md new file mode 100644 index 00000000..ff7e74e2 --- /dev/null +++ b/experiments/docs/RUNBOOK.md @@ -0,0 +1,80 @@ +# Запуск и повтор + +[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md) + +## Подготовка + +Команды выполнять в **WSL/Linux из корня лабораторной**. Для закреплённого окружения анализа нужны Python 3.12, модуль `venv`, Docker с cgroup v2/`memory.peak` и Compose v2+. Ориентир: 4 CPU, 6 ГиБ RAM Docker, 10 ГиБ диска. Порты 18080, 19000, 18404 должны быть свободны. + +```sh +bash experiments/setup.sh +.venv/bin/python -m unittest discover -s experiments/tests -v +.venv/bin/python experiments/run.py check +``` + +`setup.sh` создаёт `.venv` в корне и устанавливает pandas/matplotlib из [requirements.txt](../requirements.txt). Версии прямых и транзитивных зависимостей закреплены. Они нужны только для графиков; runner использует стандартную библиотеку (Python 3.9+). `.venv` уже исключён из Git. Если `venv` недоступен: `sudo apt install python3-venv`. + +Не менять исходники между prepare и концом серии, не запускать параллельно сборки/тесты/нагрузку. Рабочий стенд не останавливается автоматически; оставь его без нагрузки. + +## Пилот + +```sh +.venv/bin/python experiments/run.py prepare --pilot +# Подставь путь из строки Prepared: +PILOT=evidence/upload-comparison/<каталог-pilot> +.venv/bin/python experiments/run.py run "$PILOT" +.venv/bin/python experiments/run.py summarize "$PILOT" +.venv/bin/python experiments/plot.py "$PILOT" +``` + +Пилот: 2 moderate-профиля, 2 клиента, 1 волна, 1 парный повтор — 4 прогона и 8 измеряемых загрузок. Он проверяет инструменты и не участвует в H1. При исправлении инструментов сохрани старые данные и подготовь новую серию. + +## Основная серия + +```sh +.venv/bin/python experiments/run.py prepare --reuse-images "$PILOT" +CAMPAIGN=evidence/upload-comparison/<каталог-formal> +.venv/bin/python experiments/run.py run "$CAMPAIGN" +.venv/bin/python experiments/run.py summarize "$CAMPAIGN" +.venv/bin/python experiments/plot.py "$CAMPAIGN" +``` + +При изменении исходников или без пилота используй `prepare` без `--reuse-images`. Подготовка собирает собственные образы и фиксирует их ID, но не выполняет загрузки. Основная серия: 24 прогона, 720 измеряемых загрузок. + +`run`: код 0 — план завершён, 1 — сбой окружения/оркестратора, 130 — отмена. Решение о H1 находится в `metrics.json → decision`, а не в коде выхода. + +## Результаты + +Весь каталог серии сохраняется в **корневом `evidence/upload-comparison/`**: + +- `protocol.json`, `plan.json`, `seal.json`, исходники, fixtures и image ID — условия повторения. +- `cases/*` — сырые запросы, RAM-точки, счётчики, проверки оригиналов, состояния и ошибки. +- `metrics.json` / `metrics.csv` — итоговые показатели и критерии; `analysis-provenance.json` — версия анализа. +- `plots/<время>/` — PNG-графики трафика, отношения direct/proxy, RAM, p95 и ошибок; `provenance.json` фиксирует входы и версии библиотек. + +Графики показывают все доступные прогоны; пропуски не заменяются нулями. Качество измерений и итог H1 проверяй по JSON. Для разбора передай весь каталог, включая неудачные прогоны. `/tmp/highload-upload-exp-*` содержит credentials: его не архивировать. + +## Повтор, остановка и уборка + +Новый полный повтор без перезаписи: + +```sh +.venv/bin/python experiments/run.py prepare --reuse-images "$CAMPAIGN" +# Подставь новый путь и повтори run → summarize → plot. +``` + +Один `Ctrl+C` сохраняет данные и запускает уборку. Активные запросы завершаются в пределах таймаута. Продолжить только ещё не начатые прогоны: + +```sh +.venv/bin/python experiments/run.py run "$CAMPAIGN" --resume +``` + +Прерванные попытки остаются в evidence и могут дать `inconclusive`; для полного повторения нужна новая серия. Не менять seal/state вручную. + +`run ... --keep-volumes` оставляет экспериментальные volumes для отладки; при resume сохрани тот же флаг. После разбора, аварийного завершения WSL или ошибки уборки: + +```sh +.venv/bin/python experiments/run.py cleanup "$CAMPAIGN" +``` + +Удаляются только ресурсы выбранной серии, evidence остаётся. При обычном завершении уборка автоматическая. Исходники и документы входят в manifest: их изменение требует нового prepare. Для summarize старой серии `analysis.py` и `common.py` должны совпадать с архивом `source.tar.gz`; старые образы сохраняй через `docker image save/load`. diff --git a/experiments/plot.py b/experiments/plot.py new file mode 100644 index 00000000..f1e3e54c --- /dev/null +++ b/experiments/plot.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Plot saved summaries without changing raw evidence or the H1 decision.""" +import argparse +import datetime +import hashlib +import json +import platform +import sys +from pathlib import Path + +import matplotlib +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parent / "src")) +from common import campaign_path + + +def sha256(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def create_plots(campaign): + inputs = [campaign / "metrics.csv", campaign / "metrics.json"] + frame = pd.read_csv(inputs[0]) + summary = json.loads(inputs[1].read_text()) + protocol = json.loads((campaign / "protocol.json").read_text()) + profiles = [p["name"] for p in protocol["profiles"]] + status = summary["decision"]["criterion_status"] + output = campaign / "plots" / datetime.datetime.now( + datetime.timezone.utc + ).strftime("%Y%m%dT%H%M%S.%fZ") + output.mkdir(parents=True) + + def finish(fig, ax, title, ylabel, name): + ax.set_title(title + "\n" + status) + ax.set_ylabel(ylabel) + ax.set_xticks(range(len(profiles)), profiles, rotation=20, ha="right") + ax.grid(axis="y", alpha=0.25) + ax.legend(loc="best", fontsize=8) + fig.tight_layout() + fig.savefig(output / (name + ".png"), dpi=160) + plt.close(fig) + + def points(column, scale, title, ylabel, name, limit=None): + fig, ax = plt.subplots(figsize=(10, 5)) + for mode, offset, color in [("proxy", -0.12, "#2563eb"), + ("direct", 0.12, "#d97706")]: + subset = frame[frame["mode"] == mode] + x, y = [], [] + for index, profile in enumerate(profiles): + values = pd.to_numeric( + subset.loc[subset["profile"] == profile, column], errors="coerce" + ).dropna().tolist() + for j, value in enumerate(values): + x.append(index + offset + (j - (len(values) - 1) / 2) * 0.035) + y.append(value / scale) + ax.scatter(x, y, label=mode, color=color, alpha=0.85) + if limit is not None: + ax.axhline(limit, color="red", linestyle="--", label="API limit") + finish(fig, ax, title + " (each point = one run)", ylabel, name) + + points("api_bytes_per_ready", 1024 ** 2, "API RX+TX per ready attachment", + "MiB / ready", "api-traffic") + points("api_sampled_peak_window_max", 1024 ** 2, + "API sampled memory peak: maximum of two replicas", "MiB", + "api-memory-window", limit=512) + points("api_peak_lifetime_max", 1024 ** 2, + "API lifetime memory peak (includes startup and warmup)", "MiB", + "api-memory-lifetime", limit=512) + points("p95_seconds", 1, "Successful upload flow p95", "seconds", "latency-p95") + points("errors", 1, "Upload errors (missing runs are absent)", "count", "errors") + + fig, ax = plt.subplots(figsize=(10, 5)) + pairs = summary["decision"].get("pairs", []) + for valid, marker, label in [(True, "o", "instrumentation valid"), + (False, "x", "instrumentation issues")]: + selected = [p for p in pairs if p["instrumentation_valid"] == valid] + ax.scatter([profiles.index(p["profile"]) + + (p["repetition"] - 2) * 0.06 for p in selected], + [p["direct_proxy_ratio"] for p in selected], + marker=marker, label=label) + ax.axhline(protocol["criteria"]["median_direct_proxy_traffic_ratio_max"], + linestyle="--", color="green", label="median threshold") + ax.axhline(protocol["criteria"]["each_pair_traffic_ratio_max"], + linestyle=":", color="red", label="each pair threshold") + finish(fig, ax, "Paired API traffic ratio (all available pairs)", + "direct / proxy", "traffic-ratio") + provenance = { + "created_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "python": platform.python_version(), "pandas": pd.__version__, + "matplotlib": matplotlib.__version__, + "inputs_sha256": {f.name: sha256(f) for f in inputs + [campaign / "protocol.json"]}, + "plot_script_sha256": sha256(Path(__file__)), + "requirements_sha256": sha256(Path(__file__).parent / "requirements.txt"), + "criterion_status": status, + "note": "All available runs shown; missing values omitted, never replaced with zero. " + "Instrumentation validity and system failures must be read in metrics.json.", + } + (output / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n") + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("campaign", help="prepared campaign under root evidence/upload-comparison") + args = parser.parse_args() + try: + print(create_plots(campaign_path(args.campaign))) + except (OSError, ValueError, KeyError) as exc: + parser.exit(1, "Cannot plot: check campaign path and run summarize first (" + + type(exc).__name__ + ").\n") + + +if __name__ == "__main__": + main() diff --git a/experiments/requirements.txt b/experiments/requirements.txt new file mode 100644 index 00000000..f69f10e3 --- /dev/null +++ b/experiments/requirements.txt @@ -0,0 +1,15 @@ +# Analysis environment (Python 3.12). +contourpy==1.4.0 +cycler==0.12.1 +fonttools==4.67.0 +kiwisolver==1.5.1 +matplotlib==3.9.2 +numpy==2.5.4 +packaging==26.3 +pandas==2.2.3 +pillow==13.0.0 +pyparsing==3.3.3 +python-dateutil==2.9.0.post0 +pytz==2026.5 +six==1.17.0 +tzdata==2026.5 diff --git a/experiments/run.py b/experiments/run.py new file mode 100644 index 00000000..76f0ad9b --- /dev/null +++ b/experiments/run.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +"""WSL/Linux entry point. Nothing is measured until the explicit run command.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parent / "src")) +from driver import main + +if __name__ == "__main__": + main() diff --git a/experiments/setup.sh b/experiments/setup.sh new file mode 100644 index 00000000..26e7fa85 --- /dev/null +++ b/experiments/setup.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +PYTHON="${PYTHON:-python3}" +"$PYTHON" -c 'import sys; sys.exit("Use Python 3.12 for the pinned analysis environment") if sys.version_info[:2] != (3, 12) else None' +"$PYTHON" -m venv "$ROOT/.venv" +"$ROOT/.venv/bin/python" -m pip install --no-compile -r "$ROOT/experiments/requirements.txt" +"$ROOT/.venv/bin/python" -m pip check +printf '\nReady: %s/.venv/bin/python\n' "$ROOT" diff --git a/experiments/src/analysis.py b/experiments/src/analysis.py new file mode 100644 index 00000000..2ecac644 --- /dev/null +++ b/experiments/src/analysis.py @@ -0,0 +1,392 @@ +import csv +import json +import math +import statistics + +from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save + + +def percentile(values, fraction): + if not values: + return None + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] + + +def json_lines(path): + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] + + +def case_metrics(directory, protocol): + case = load(directory / "case.json") + outcome = ( + load(directory / "outcome.json") + if (directory / "outcome.json").exists() + else {"status": "interrupted"} + ) + requests, verification = ( + json_lines(directory / "requests.jsonl"), + json_lines(directory / "verification.jsonl"), + ) + expected = protocol["clients"] * protocol["waves"] + result = { + **case, + "status": outcome["status"], + "expected_requests": expected, + "requests_recorded": len(requests), + "ready": sum(r.get("ready", False) for r in requests), + "errors": sum( + not r.get("ready", False) and r.get("error") != "interrupted" + for r in requests + ), + "cancelled": sum(r.get("error") == "interrupted" for r in requests), + "verified": sum(v.get("verified", False) for v in verification), + "system_failures": [], + "instrumentation_issues": [], + "containers": [], + } + system, issues = result["system_failures"], result["instrumentation_issues"] + if result["errors"]: + system.append("upload_errors") + if ( + len({r["attachment"]["id"] for r in requests if r.get("ready")}) + != result["ready"] + ): + system.append("duplicate_attachment_ids") + if any( + not v.get("verified", False) and v.get("error") != "interrupted" + for v in verification + ): + system.append("original_verification_failed") + warmups = json_lines(directory / "warmup.jsonl") + if any( + not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups + ): + system.append("warmup_upload_errors") + if len(requests) != expected or result["verified"] != result["ready"]: + issues.append("missing_request_or_verification_records") + if outcome["status"] != "completed": + issues.append("case_" + outcome["status"]) + if outcome.get("sampler_errors"): + issues.append("sampler_error") + before = ( + load(directory / "containers-before.json") + if (directory / "containers-before.json").exists() + else [] + ) + after = ( + load(directory / "containers-after.json") + if (directory / "containers-after.json").exists() + else [] + ) + for c in after: + previous = next((p for p in before if p["id"] == c["id"]), None) + if ( + c["oom_killed"] + or c["status"] != "running" + or ( + previous + and ( + c["restarts"] != previous["restarts"] + or c["started_at"] != previous["started_at"] + ) + ) + ): + system.append("container_oom_restart_or_exit:" + c["service"]) + if not (directory / "measurement.json").exists(): + issues.append("missing_measurement") + return result + m = load(directory / "measurement.json") + seconds = (m["finished_ns"] - m["started_ns"]) / 1e9 + result.update( + window_seconds=seconds, + generator_cpu_cores=m["generator_cpu_seconds"] / seconds, + ready_rps=result["ready"] / seconds, + ) + durations = [ + (r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready") + ] + result["latency_seconds"] = { + "p50": percentile(durations, 0.50), + "p95": percentile(durations, 0.95), + "p99": percentile(durations, 0.99), + } + spreads = [] + for wave in range(protocol["waves"]): + starts = [r["started_ns"] for r in requests if r["wave"] == wave] + if len(starts) == protocol["clients"]: + spreads.append((max(starts) - min(starts)) / 1e9) + result["max_start_spread_seconds"] = max(spreads, default=None) + criteria = protocol["criteria"] + if len(spreads) != protocol["waves"] or any( + s > criteria["start_spread_seconds_max"] for s in spreads + ): + issues.append("burst_start_spread") + if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]: + issues.append("possible_generator_cpu_bottleneck") + instances = {r["attachment"]["instance"] for r in requests if r.get("ready")} + if len(instances) != protocol["resources"]["api_replicas"]: + issues.append("not_all_api_replicas_exercised") + warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")} + if len(warm_instances) != protocol["resources"]["api_replicas"]: + issues.append("not_all_api_replicas_warmed") + samples = json_lines(directory / "samples.jsonl") + api_traffic, background = 0, 0.0 + for c in before: + cid = c["id"] + if cid not in m["before"] or cid not in m["after"]: + issues.append("missing_counter_snapshot:" + c["service"]) + continue + start, end = m["before"][cid], m["after"][cid] + points = [ + s + for s in samples + if s["container"] == cid + and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"] + ] + times = ( + [m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]] + ) + gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])] + coverage = len(points) * protocol["sample_interval_seconds"] / seconds + rx, tx = ( + end["rx_bytes"] - start["rx_bytes"], + end["tx_bytes"] - start["tx_bytes"], + ) + if rx < 0 or tx < 0: + issues.append("network_counter_reset:" + c["service"]) + idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid] + idle_seconds = ( + idle_end["read_started_ns"] - idle_start["read_started_ns"] + ) / 1e9 + idle_rate = ( + (idle_end["rx_bytes"] + idle_end["tx_bytes"]) + - (idle_start["rx_bytes"] + idle_start["tx_bytes"]) + ) / idle_seconds + cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"] + peak = max( + [s["memory_current"] for s in points] + + [start["memory.current"], end["memory.current"]] + ) + events = { + key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0) + for key in ("oom", "oom_kill", "max") + } + if end["memory.events"].get("oom", 0) or end["memory.events"].get( + "oom_kill", 0 + ): + system.append("oom_event:" + c["service"]) + if ( + coverage < criteria["sampler_coverage_fraction_min"] + or max(gaps) > criteria["sampler_gap_seconds_max"] + ): + issues.append("sampler_coverage:" + c["service"]) + item = { + "service": c["service"], + "hostname": c["hostname"], + "rx_bytes": rx, + "tx_bytes": tx, + "cpu_seconds": cpu_usec / 1e6, + "cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9), + "throttled_seconds": ( + end["cpu.stat"].get("throttled_usec", 0) + - start["cpu.stat"].get("throttled_usec", 0) + ) + / 1e6, + "throttled_periods": end["cpu.stat"].get("nr_throttled", 0) + - start["cpu.stat"].get("nr_throttled", 0), + "baseline_memory": start["memory.current"], + "sampled_peak_window": peak, + "peak_lifetime_before": start["memory.peak"], + "peak_lifetime_after": end["memory.peak"], + "memory_limit": end["memory.max"], + "memory_events_delta": events, + "idle_bytes_per_second": idle_rate, + "sample_count": len(points), + "sample_coverage": coverage, + "sample_gap_max_seconds": max(gaps), + } + result["containers"].append(item) + if c["service"] == "api": + api_traffic += rx + tx + background += idle_rate * seconds + if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]: + system.append("api_lifetime_peak_reached_limit") + if sum(c["service"] == "api" for c in result["containers"]) != 2: + issues.append("missing_api_counters") + result["api_rx_tx_bytes"] = api_traffic + result["api_bytes_per_ready"] = ( + api_traffic / result["ready"] if result["ready"] else None + ) + api_metrics = [c for c in result["containers"] if c["service"] == "api"] + result["api_sampled_peak_window_max"] = max( + (c["sampled_peak_window"] for c in api_metrics), default=None + ) + result["api_peak_lifetime_max"] = max( + (c["peak_lifetime_after"] for c in api_metrics), default=None + ) + result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics) + result["s3_rx_tx_bytes"] = sum( + c["rx_bytes"] + c["tx_bytes"] + for c in result["containers"] + if c["service"] == "s3" + ) + result["estimated_background_fraction"] = background / ( + expected * case["size_bytes"] + ) + if ( + result["estimated_background_fraction"] + > criteria["idle_background_payload_fraction_max"] + ): + issues.append("excessive_background_traffic") + result["system_failures"] = sorted(set(system)) + result["instrumentation_issues"] = sorted(set(issues)) + return result + + +def decision(protocol, cases, plan): + pairs, issues, failures, profiles = [], [], [], [] + if len(cases) != len(plan): + issues.append("not_all_planned_cases_finished") + for c in cases: + failures.extend(c["system_failures"]) + issues.extend(c["instrumentation_issues"]) + for profile in protocol["profiles"]: + valid_ratios = [] + for repetition in range(1, protocol["repetitions"] + 1): + pair = [ + c + for c in cases + if c["profile"] == profile["name"] and c["repetition"] == repetition + ] + modes = {c["mode"]: c for c in pair} + if set(modes) != {"proxy", "direct"} or not all( + m.get("api_bytes_per_ready") for m in modes.values() + ): + issues.append("incomplete_pair:" + profile["name"]) + continue + ratio = ( + modes["direct"]["api_bytes_per_ready"] + / modes["proxy"]["api_bytes_per_ready"] + ) + valid = all(not m["instrumentation_issues"] for m in modes.values()) + if valid: + valid_ratios.append(ratio) + pairs.append( + { + "profile": profile["name"], + "repetition": repetition, + "direct_proxy_ratio": ratio, + "instrumentation_valid": valid, + } + ) + profiles.append( + { + "profile": profile["name"], + "valid_pairs": len(valid_ratios), + "median_traffic_ratio": statistics.median(valid_ratios) + if valid_ratios + else None, + "min_traffic_ratio": min(valid_ratios, default=None), + "max_traffic_ratio": max(valid_ratios, default=None), + } + ) + if any( + r > protocol["criteria"]["each_pair_traffic_ratio_max"] + for r in valid_ratios + ) or ( + len(valid_ratios) == protocol["repetitions"] + and statistics.median(valid_ratios) + > protocol["criteria"]["median_direct_proxy_traffic_ratio_max"] + ): + failures.append("traffic_threshold:" + profile["name"]) + if protocol["purpose"] == "pilot": + status = "pilot_not_for_hypothesis" + elif failures: + # OOM, errors and incorrect bytes are contrary outcomes, never exclusions. + status = "rejected" + elif issues: + status = "inconclusive" + else: + status = "confirmed_in_tested_conditions" + return { + "criterion_status": status, + "system_or_threshold_failures": sorted(set(failures)), + "instrumentation_issues": sorted(set(issues)), + "pairs": pairs, + "profiles": profiles, + } + + +def summarize(campaign): + manifest = load(campaign / "source-manifest.json") + analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"] + if any( + manifest.get(str(path.relative_to(ROOT))) != digest(path) + for path in analysis_sources + ): + raise ExperimentError( + "analysis code differs from frozen snapshot; use the archived experiment source" + ) + protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json") + cases = [] + for c in plan: + directory = campaign / "cases" / c["directory"] + if (directory / "case.json").exists(): + cases.append(case_metrics(directory, protocol)) + result = { + "protocol_version": protocol["version"], + "purpose": protocol["purpose"], + "decision": decision(protocol, cases, plan), + "cases": cases, + } + if (campaign / "execution-failure.json").exists(): + result["decision"]["instrumentation_issues"].append( + "campaign_execution_failure" + ) + if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions": + result["decision"]["criterion_status"] = "inconclusive" + save(campaign / "metrics.json", result) + save( + campaign / "analysis-provenance.json", + { + "source_sha256": { + str(p.relative_to(ROOT)): digest(p) for p in analysis_sources + }, + "seal_sha256": digest(campaign / "seal.json"), + }, + ) + with (campaign / "metrics.csv").open("w", newline="") as output: + columns = [ + "profile", + "mode", + "repetition", + "ready", + "errors", + "verified", + "api_bytes_per_ready", + "api_sampled_peak_window_max", + "api_peak_lifetime_max", + "api_cpu_seconds_total", + "s3_rx_tx_bytes", + "window_seconds", + "ready_rps", + "generator_cpu_cores", + "p50_seconds", + "p95_seconds", + "p99_seconds", + ] + writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore") + writer.writeheader() + writer.writerows( + { + **case, + **{ + name + "_seconds": value + for name, value in case.get("latency_seconds", {}).items() + }, + } + for case in cases + ) + return result diff --git a/experiments/src/client.py b/experiments/src/client.py new file mode 100644 index 00000000..89cb9397 --- /dev/null +++ b/experiments/src/client.py @@ -0,0 +1,273 @@ +import base64 +import hashlib +import http.client +import json +import secrets +import time +from urllib.parse import urlsplit + + +class FlowError(Exception): + def __init__(self, code): + self.code = code + + +class Client: + def __init__(self, base, protocol, stop): + self.base, self.protocol, self.stop = base, protocol, stop + + def request( + self, + method, + url, + body=None, + token=None, + headers=None, + stage="api", + records=None, + deadline=None, + ): + if self.stop.is_set(): + raise FlowError("interrupted") + parsed = urlsplit(url) + if parsed.scheme != "http" or parsed.hostname not in ("localhost", "127.0.0.1"): + raise FlowError("unexpected_endpoint") + expected_port = self.protocol["ports"]["s3" if stage == "s3_post" else "api"] + if parsed.port != expected_port: + raise FlowError("unexpected_endpoint_port") + data = json.dumps(body).encode("utf-8") if isinstance(body, dict) else body + h = dict(headers or {}) + if isinstance(body, dict): + h["Content-Type"] = "application/json" + if token: + h["Authorization"] = "Bearer " + token + remaining = ( + deadline - time.monotonic() + if deadline + else self.protocol["request_timeout_seconds"] + ) + if remaining <= 0: + raise FlowError("flow_timeout") + connection = http.client.HTTPConnection( + parsed.hostname, + parsed.port, + timeout=min(self.protocol["request_timeout_seconds"], remaining), + ) + started = time.monotonic_ns() + record = { + "stage": stage, + "method": method, + "started_ns": started, + "request_body_bytes": len(data or b""), + } + if records is not None: + records.append(record) + try: + connection.request( + method, + parsed.path + (("?" + parsed.query) if parsed.query else ""), + data, + h, + ) + response = connection.getresponse() + payload = response.read(65537) + record.update( + status=response.status, + response_body_bytes=len(payload), + instance=response.getheader("X-Instance-Id"), + request_id=response.getheader("X-Request-Id"), + ) + if len(payload) > 65536: + raise FlowError("unexpected_response_size") + try: + decoded = json.loads(payload) if payload else None + except ValueError: + if response.status < 400: + raise FlowError("unexpected_response_format") from None + decoded = None + if response.status >= 400: + code = ( + decoded.get("error", {}).get("code", "http_error") + if isinstance(decoded, dict) + else "http_" + str(response.status) + ) + record["error"] = code + raise FlowError(code) + return decoded, response.status, record + except FlowError: + raise + except Exception as error: + record["error"] = type(error).__name__ + raise FlowError(type(error).__name__) from None + finally: + record["finished_ns"] = time.monotonic_ns() + connection.close() + + def api(self, method, path, **kwargs): + return self.request(method, self.base + path, **kwargs) + + def bootstrap(self): + actors = [] + for label in ("alice", "bob"): + username = "exp_" + label + "_" + secrets.token_hex(6) + password = secrets.token_hex(24) + user, status, _ = self.api( + "POST", + "/api/v1/users", + body={ + "username": username, + "display_name": label, + "password": password, + }, + ) + if status != 201: + raise FlowError("registration_status") + basic = base64.b64encode((username + ":" + password).encode()).decode() + session, status, _ = self.api( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + basic}, + ) + if status != 201: + raise FlowError("session_status") + actors.append({"id": user["id"], "token": session["token"]}) + chat, status, _ = self.api( + "POST", + "/api/v1/chats", + token=actors[0]["token"], + body={ + "title": "Upload experiment", + "member_ids": [a["id"] for a in actors], + }, + ) + if status != 201: + raise FlowError("chat_status") + return actors, chat["id"] + + def upload(self, mode, actor, chat, data, expected_digest, records, deadline): + path = "/api/v1/chats/" + chat + if mode == "proxy": + attachment, status, record = self.api( + "POST", + path + "/attachments?filename=experiment.png", + body=data, + token=actor["token"], + headers={"Content-Type": "image/png"}, + stage="proxy", + records=records, + deadline=deadline, + ) + else: + # Matches the browser: hash inside the measured logical direct flow. + start_hash = time.monotonic_ns() + checksum = hashlib.sha256(data).hexdigest() + records.append( + { + "stage": "client_sha256", + "started_ns": start_hash, + "finished_ns": time.monotonic_ns(), + } + ) + instruction, status, _ = self.api( + "POST", + path + "/attachment-uploads", + token=actor["token"], + body={ + "filename": "experiment.png", + "content_type": "image/png", + "size_bytes": len(data), + "sha256": checksum, + }, + stage="init", + records=records, + deadline=deadline, + ) + if status != 201: + raise FlowError("init_status") + fields = instruction["upload"]["fields"] + boundary = "experiment-" + secrets.token_hex(12) + pieces = [ + f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode() + for k, v in fields.items() + ] + pieces += [ + f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="experiment.png"\r\nContent-Type: image/png\r\n\r\n'.encode(), + data, + f"\r\n--{boundary}--\r\n".encode(), + ] + multipart = b"".join(pieces) + # This is the only S3 request; no Bearer token or application headers. + _, post_status, _ = self.request( + "POST", + instruction["upload"]["url"], + body=multipart, + headers={"Content-Type": "multipart/form-data; boundary=" + boundary}, + stage="s3_post", + records=records, + deadline=deadline, + ) + if post_status != 204: + raise FlowError("s3_post_status") + attachment, status, record = self.api( + "POST", + path + "/attachment-uploads/" + instruction["upload_id"] + "/complete", + token=actor["token"], + stage="complete", + records=records, + deadline=deadline, + ) + if ( + status != 201 + or attachment.get("status") != "ready" + or attachment.get("size_bytes") != len(data) + or attachment.get("sha256") != expected_digest + ): + raise FlowError("incorrect_ready_metadata") + return { + "id": attachment["id"], + "sha256": attachment["sha256"], + "size_bytes": attachment["size_bytes"], + "instance": record["instance"], + } + + def verify(self, actor, attachment, size, expected_digest): + if self.stop.is_set(): + raise FlowError("interrupted") + parsed = urlsplit(self.base) + connection = http.client.HTTPConnection( + parsed.hostname, parsed.port, timeout=35 + ) + checksum, received = hashlib.sha256(), 0 + try: + connection.request( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content", + headers={"Authorization": "Bearer " + actor["token"]}, + ) + response = connection.getresponse() + if ( + response.status != 200 + or response.getheader("Content-Length") != str(size) + or response.getheader("Content-Type") != "image/png" + ): + raise FlowError("download_headers") + while True: + if self.stop.is_set(): + raise FlowError("interrupted") + block = response.read(32768) + if not block: + break + received += len(block) + if received > size: + raise FlowError("download_size") + checksum.update(block) + if received != size or checksum.hexdigest() != expected_digest: + raise FlowError("download_checksum") + return { + "id": attachment["id"], + "verified": True, + "size_bytes": received, + "sha256": checksum.hexdigest(), + } + finally: + connection.close() diff --git a/experiments/src/common.py b/experiments/src/common.py new file mode 100644 index 00000000..85ca6bcf --- /dev/null +++ b/experiments/src/common.py @@ -0,0 +1,112 @@ +import hashlib +import json +import os +import subprocess +import tempfile +from pathlib import Path + +EXPERIMENT = Path(__file__).resolve().parents[1] +ROOT = EXPERIMENT.parent +EVIDENCE = ROOT / "evidence" / "upload-comparison" + + +class ExperimentError(RuntimeError): + pass + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def save(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text( + json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + os.replace(temporary, path) + + +def load(path): + return json.loads(path.read_text(encoding="utf-8")) + + +def command(args, log=None, timeout=300, input_text=None): + # Never format the exception with subprocess output: Docker can include + # environment details. Explicit build/start logs use commands without secrets. + result = subprocess.run( + args, + cwd=ROOT, + text=True, + input=input_text, + stdout=log or subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + ) + if result.returncode: + raise ExperimentError( + "command failed: " + args[0] + " (exit " + str(result.returncode) + ")" + ) + return result.stdout.strip() if log is None else "" + + +def source_files(): + fixed = [ + ROOT / "src/Dockerfile", + ROOT / "src/.dockerignore", + ROOT / "src/go.mod", + ROOT / "src/go.sum", + ROOT / "infra/minio.Dockerfile", + ROOT / "infra/haproxy.cfg", + ] + app = sorted( + p + for p in (ROOT / "src/backend").rglob("*") + if p.is_file() and p.suffix in (".go", ".sql") + ) + harness = sorted( + p + for p in EXPERIMENT.rglob("*") + if p.is_file() + and "evidence" not in p.relative_to(EXPERIMENT).parts + and "__pycache__" not in p.parts + and p.suffix != ".pyc" + ) + return sorted(set(fixed + app + harness)) + + +def source_manifest(): + return {str(p.relative_to(ROOT)): digest(p) for p in source_files()} + + +def check_sources(campaign): + expected = load(campaign / "source-manifest.json") + if source_manifest() != expected: + raise ExperimentError( + "source/config/docs changed since prepare; create a new campaign" + ) + + +def check_seal(campaign): + seal = load(campaign / "seal.json") + for name, expected in seal.items(): + if digest(campaign / name) != expected: + raise ExperimentError("frozen artifact changed: " + name) + for entry in load(campaign / "fixtures.json"): + if digest(campaign / "fixtures" / entry["filename"]) != entry["sha256"]: + raise ExperimentError("fixture checksum changed") + + +def campaign_path(value): + path = Path(value).expanduser().resolve() + if path.parent != EVIDENCE.resolve() or not (path / "protocol.json").is_file(): + raise ExperimentError( + "campaign must be a prepared directory directly inside evidence/upload-comparison" + ) + return path + + +def private_directory(): + path = Path(tempfile.mkdtemp(prefix="highload-upload-exp-", dir="/tmp")) + path.chmod(0o700) + return path diff --git a/experiments/src/driver.py b/experiments/src/driver.py new file mode 100644 index 00000000..4efc6ff7 --- /dev/null +++ b/experiments/src/driver.py @@ -0,0 +1,833 @@ +import argparse +import concurrent.futures +import datetime +import fcntl +import hashlib +import json +import os +import platform +import resource +import secrets +import shutil +import signal +import socket +import subprocess +import sys +import tarfile +import threading +import time +import zlib +from pathlib import Path + +from analysis import summarize +from client import Client, FlowError +from common import ( + EVIDENCE, + EXPERIMENT, + ROOT, + ExperimentError, + campaign_path, + check_seal, + check_sources, + command, + digest, + load, + private_directory, + save, + source_manifest, +) +from fixtures import make_png +from telemetry import Sampler, inspect_containers, snapshot + + +def make_plan(protocol): + blocks = [] + for index, profile in enumerate(protocol["profiles"]): + for repetition in range(1, protocol["repetitions"] + 1): + modes = ( + ["proxy", "direct"] if (index + repetition) % 2 else ["direct", "proxy"] + ) + blocks.append((profile, repetition, modes)) + blocks.sort( + key=lambda b: hashlib.sha256( + f"{protocol['seed']}:{b[0]['name']}:{b[1]}".encode() + ).digest() + ) + plan = [] + for profile, repetition, modes in blocks: + for mode in modes: + number = len(plan) + 1 + plan.append( + { + "number": number, + "profile": profile["name"], + "mode": mode, + "repetition": repetition, + "size_bytes": profile["size_bytes"], + "directory": f"{number:02d}-{profile['name']}-{mode}-r{repetition}", + } + ) + return plan + + +def host_info(): + info = json.loads(command(["docker", "info", "--format", "{{json .}}"], timeout=30)) + if str(info.get("CgroupVersion")) != "2": + raise ExperimentError("Docker cgroup v2 is required") + safe = { + key: info.get(key) + for key in ( + "ServerVersion", + "CgroupVersion", + "NCPU", + "MemTotal", + "KernelVersion", + "OperatingSystem", + "Driver", + ) + } + return { + "python": sys.version, + "zlib": zlib.ZLIB_VERSION, + "platform": platform.platform(), + "docker": safe, + "compose": command(["docker", "compose", "version", "--short"]), + "client_cpu_affinity_count": len(os.sched_getaffinity(0)) + if hasattr(os, "sched_getaffinity") + else os.cpu_count(), + "disk_free_bytes": shutil.disk_usage(EXPERIMENT).free, + } + + +def check_ports(protocol): + for port in protocol["ports"].values(): + with socket.socket() as test: + try: + test.bind(("127.0.0.1", port)) + except OSError: + raise ExperimentError( + "experiment port is occupied: " + str(port) + ) from None + + +def preflight(): + if platform.system() != "Linux": + raise ExperimentError("run inside WSL Ubuntu/Linux, not Windows Python") + protocol = load(EXPERIMENT / "config/protocol.json") + validate_compose(protocol) + info = host_info() + check_ports(protocol) + if not command(["docker", "compose", "version", "--short"]): + raise ExperimentError("Docker Compose plugin is required") + return info + + +def validate_compose(protocol): + if len(set(protocol["ports"].values())) != 3: + raise ExperimentError("API/S3/stats ports must differ") + values = { + "REPO_ROOT": str(ROOT), + "SECRET_DIR": "/tmp/experiment-config-check", + **{ + role + "_IMAGE": "sha256:" + "0" * 64 + for role in ("APP", "S3", "DB", "REDIS", "HAPROXY") + }, + **{ + role.upper() + "_PORT": str(port) + for role, port in protocol["ports"].items() + }, + } + response = subprocess.run( + [ + "docker", + "compose", + "--env-file", + "/dev/null", + "--project-name", + "hl-upload-exp-config-check", + "-f", + str(EXPERIMENT / "config/compose.yml"), + "config", + "--format", + "json", + ], + check=False, + cwd=ROOT, + env={**os.environ, **values}, + text=True, + capture_output=True, + timeout=30, + ) + if response.returncode: + raise ExperimentError("experiment Compose config is invalid") + services = json.loads(response.stdout)["services"] + resources = protocol["resources"] + for role in ("api", "db", "s3", "redis", "haproxy"): + if ( + float(services[role]["cpus"]) != resources[role + "_cpus"] + or int(services[role]["mem_limit"]) != resources[role + "_memory_bytes"] + ): + raise ExperimentError("protocol/Compose resource mismatch: " + role) + if int(services[role]["memswap_limit"]) != int(services[role]["mem_limit"]): + raise ExperimentError("experiment swap must be disabled: " + role) + api = services["api"] + expected_env = { + "GOMAXPROCS": str(resources["api_gomaxprocs"]), + "GOMEMLIMIT": resources["api_go_memory_limit"], + "DATABASE_MAX_CONNS": str(resources["database_max_connections_per_replica"]), + } + if api["deploy"]["replicas"] != resources["api_replicas"] or any( + api["environment"].get(k) != v for k, v in expected_env.items() + ): + raise ExperimentError("protocol/Compose API environment mismatch") + if ( + api["tmpfs"] != ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"] + or resources["api_tmpfs_bytes"] != 16777216 + ): + raise ExperimentError("unexpected tmpfs configuration") + + +def prepare(pilot=False, reuse=None): + info = preflight() + EVIDENCE.mkdir(parents=True, exist_ok=True) + stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") + campaign = EVIDENCE / ( + stamp + "-" + ("pilot" if pilot else "formal") + "-" + secrets.token_hex(3) + ) + campaign.mkdir() + protocol = load(EXPERIMENT / "config/protocol.json") + if pilot: + protocol.update( + purpose="pilot", repetitions=1, clients=2, waves=1, warmup_clients=2 + ) + protocol["profiles"] = [p for p in protocol["profiles"] if p["width"] == 1024] + # Frozen before builds or any application requests. + save(campaign / "protocol.json", protocol) + save(campaign / "plan.json", make_plan(protocol)) + sources = source_manifest() + save(campaign / "source-manifest.json", sources) + shutil.copyfile(EXPERIMENT / "config/compose.yml", campaign / "compose.yml") + with tarfile.open(campaign / "source.tar.gz", "w:gz") as archive: + for name in sources: + archive.add(ROOT / name, arcname=name, recursive=False) + save(campaign / "host-prepare.json", info) + save( + campaign / "git.json", + { + "head": command(["git", "rev-parse", "HEAD"]), + "branch": command(["git", "branch", "--show-current"]), + "status": command(["git", "status", "--short", "--untracked-files=no"]), + }, + ) + fixtures = [] + for p in protocol["profiles"]: + filename = p["name"] + ".png" + data = make_png(p["width"], p["height"], p["size_bytes"], protocol["seed"]) + (campaign / "fixtures").mkdir(exist_ok=True) + (campaign / "fixtures" / filename).write_bytes(data) + fixtures.append( + {**p, "filename": filename, "sha256": hashlib.sha256(data).hexdigest()} + ) + save(campaign / "fixtures.json", fixtures) + try: + if reuse: + check_seal(reuse) + if load(reuse / "source-manifest.json") != sources: + raise ExperimentError( + "reuse-images source manifest differs; prepare without reuse" + ) + images = load(reuse / "images.json") + for image in images.values(): + command(["docker", "image", "inspect", image["id"]]) + else: + refs = dict(protocol["images"]) + app_hash = hashlib.sha256( + json.dumps( + {k: v for k, v in sources.items() if k.startswith("src/")}, + sort_keys=True, + ).encode() + ).hexdigest()[:16] + refs["app"] = "highload-upload-experiment-app:" + app_hash + refs["s3"] = ( + "highload-upload-experiment-s3:" + + sources["infra/minio.Dockerfile"][:16] + ) + with (campaign / "prepare.log").open("w") as log: + for role in ("db", "redis", "haproxy"): + try: + command(["docker", "image", "inspect", refs[role]]) + except ExperimentError: + command(["docker", "pull", refs[role]], log, timeout=600) + command( + [ + "docker", + "build", + "--tag", + refs["app"], + "--file", + str(ROOT / "src/Dockerfile"), + str(ROOT / "src"), + ], + log, + timeout=1800, + ) + command( + [ + "docker", + "build", + "--tag", + refs["s3"], + "--file", + str(ROOT / "infra/minio.Dockerfile"), + str(ROOT / "infra"), + ], + log, + timeout=1800, + ) + images = {} + for role, ref in refs.items(): + image = json.loads(command(["docker", "image", "inspect", ref]))[0] + images[role] = { + "id": image["Id"], + "reference": ref, + "repo_digests": image.get("RepoDigests", []), + "os": image["Os"], + "architecture": image["Architecture"], + "layers": image["RootFS"]["Layers"], + } + save(campaign / "images.json", images) + check_sources(campaign) + names = [ + "protocol.json", + "plan.json", + "source-manifest.json", + "source.tar.gz", + "fixtures.json", + "images.json", + "compose.yml", + ] + save(campaign / "seal.json", {name: digest(campaign / name) for name in names}) + save( + campaign / "prepared.json", + { + "status": "prepared", + "purpose": protocol["purpose"], + "cases": len(load(campaign / "plan.json")), + }, + ) + except Exception as error: + save( + campaign / "prepare-failure.json", + {"error_type": type(error).__name__, "status": "not_prepared"}, + ) + raise + print( + "Prepared; no uploads measured: " + str(campaign.relative_to(ROOT)), flush=True + ) + print("Seal: " + digest(campaign / "seal.json"), flush=True) + return campaign + + +def runtime_files(campaign): + directory = private_directory() + credentials = { + "db_password": secrets.token_hex(24), + "root_access": secrets.token_hex(12), + "root_secret": secrets.token_hex(32), + "api_access": secrets.token_hex(12), + "api_secret": secrets.token_hex(32), + } + credentials["database_url"] = ( + "postgres://messenger_exp:" + + credentials["db_password"] + + "@db:5432/messenger_exp?sslmode=disable" + ) + credentials["cache_url"] = "redis://redis:6379/0" + for name, value in credentials.items(): + path = directory / name + path.write_text(value + "\n") + path.chmod(0o644) # private parent is 0700; bind individual files for app UID. + images, protocol = load(campaign / "images.json"), load(campaign / "protocol.json") + values = { + "REPO_ROOT": str(ROOT), + "SECRET_DIR": str(directory), + **{role.upper() + "_IMAGE": image["id"] for role, image in images.items()}, + **{ + role.upper() + "_PORT": str(port) + for role, port in protocol["ports"].items() + }, + } + env = directory / "runtime.env" + env.write_text("".join(key + "=" + value + "\n" for key, value in values.items())) + env.chmod(0o600) + save(campaign / "runtime.json", {"private_directory": str(directory)}) + return directory + + +def compose(campaign, runtime, project): + if not project.startswith("hl-upload-exp-") or len(project) > 63: + raise ExperimentError("unsafe experiment project name") + return [ + "docker", + "compose", + "--project-name", + project, + "--env-file", + str(runtime / "runtime.env"), + "-f", + str(campaign / "compose.yml"), + ] + + +def safe_down(campaign, runtime, project, log, keep_volumes): + # Every deletion is scoped by a unique project label, never a global prune. + allowed = { + project_name(campaign, case["number"]) for case in load(campaign / "plan.json") + } + if project not in allowed: + raise ExperimentError("refusing cleanup of another campaign's project") + for kind in ("container", "volume", "network"): + args = [ + "docker", + kind, + "ls", + "--filter", + "label=com.docker.compose.project=" + project, + "-q", + ] + if kind == "container": + args.insert(3, "-a") + ids = command(args).splitlines() + if ids: + items = json.loads(command(["docker", kind, "inspect", *ids])) + for item in items: + labels = ( + item.get("Labels") + if kind != "container" + else item["Config"].get("Labels") + ) + if (labels or {}).get("com.docker.compose.project") != project: + raise ExperimentError( + "refusing cleanup outside the selected project" + ) + args = compose(campaign, runtime, project) + ["down", "--timeout", "15"] + if not keep_volumes: + args.append("--volumes") + command(args, log, timeout=120) + + +def project_name(campaign, number): + return ( + "hl-upload-exp-" + + campaign.name.lower().replace("t", "").replace("z", "") + + "-" + + str(number) + ) + + +def host_activity(): + result = {"loadavg": list(os.getloadavg()), "monotonic_ns": time.monotonic_ns()} + for row in Path("/proc/meminfo").read_text().splitlines(): + if row.startswith(("MemAvailable:", "SwapFree:", "SwapTotal:")): + key, value = row.split(":", 1) + result[key + "_bytes"] = int(value.split()[0]) * 1024 + return result + + +def batch(client, mode, actor, chat, data, digest_value, count, wave, output, stop): + barrier = threading.Barrier(count + 1) + + def worker(index): + records = [] + barrier.wait(timeout=15) + started = time.monotonic_ns() + result = { + "wave": wave, + "client": index, + "started_ns": started, + "ready": False, + "stages": records, + } + try: + attachment = client.upload( + mode, + actor, + chat, + data, + digest_value, + records, + time.monotonic() + client.protocol["flow_timeout_seconds"], + ) + result.update(ready=True, attachment=attachment) + except FlowError as error: + result["error"] = error.code + except Exception as error: + result["error"] = type(error).__name__ + result["finished_ns"] = time.monotonic_ns() + return result + + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool: + futures = [pool.submit(worker, index) for index in range(count)] + barrier.wait(timeout=15) + for future in concurrent.futures.as_completed(futures): + result = future.result() + output.write(json.dumps(result) + "\n") + output.flush() + results.append(result) + return results + + +def run_case(campaign, case, runtime, project, stop): + directory = campaign / "cases" / case["directory"] + directory.mkdir(parents=True) + save(directory / "case.json", {**case, "project": project}) + protocol, images = load(campaign / "protocol.json"), load(campaign / "images.json") + cmd = compose(campaign, runtime, project) + sampler, containers, outcome = ( + None, + [], + {"status": "setup_failed", "stage": "startup"}, + ) + try: + save(directory / "host-before.json", host_activity()) + with (directory / "startup.log").open("w") as log: + command( + cmd + ["up", "-d", "--no-build", "--wait", "--wait-timeout", "120"], + log, + timeout=180, + ) + ids = command(cmd + ["ps", "-q", "api", "s3"]).splitlines() + containers = inspect_containers(ids, project) + apis = [c for c in containers if c["service"] == "api"] + if len(apis) != 2 or len(containers) != 3: + raise ExperimentError("expected two API containers and one S3 container") + for c in containers: + expected = images["app" if c["service"] == "api" else "s3"]["id"] + if c["image"] != expected: + raise ExperimentError("unexpected image ID") + for c in apis: + if ( + c["memory_limit"] != protocol["resources"]["api_memory_bytes"] + or c["memory_and_swap_limit"] != c["memory_limit"] + or c["nano_cpus"] != 500000000 + or "size=16m" not in c["tmpfs"].get("/tmp/uploads", "") + ): + raise ExperimentError("API resource configuration changed") + outcome["stage"] = "bootstrap_warmup" + client = Client( + "http://127.0.0.1:" + str(protocol["ports"]["api"]), protocol, stop + ) + actors, chat = client.bootstrap() + fixture = next( + p for p in load(campaign / "fixtures.json") if p["name"] == case["profile"] + ) + data = (campaign / "fixtures" / fixture["filename"]).read_bytes() + with (directory / "warmup.jsonl").open("w") as output: + warmup = batch( + client, + case["mode"], + actors[0], + chat, + data, + fixture["sha256"], + protocol["warmup_clients"], + -1, + output, + stop, + ) + if not all(r["ready"] for r in warmup): + raise ExperimentError("warmup upload failed") + time.sleep(protocol["settle_seconds"]) + sampler = Sampler( + containers, directory / "samples.jsonl", protocol["sample_interval_seconds"] + ) + sampler.wait_ready() + idle_before = snapshot(containers) + time.sleep(protocol["idle_seconds"]) + idle_after = snapshot(containers) + save(directory / "containers-before.json", inspect_containers(ids, project)) + outcome["stage"] = "measured_uploads" + started = time.monotonic_ns() + cpu_before = resource.getrusage(resource.RUSAGE_SELF) + before = snapshot(containers) + requests = [] + with (directory / "requests.jsonl").open("w") as output: + for wave in range(protocol["waves"]): + if stop.is_set(): + break + requests.extend( + batch( + client, + case["mode"], + actors[0], + chat, + data, + fixture["sha256"], + protocol["clients"], + wave, + output, + stop, + ) + ) + if wave + 1 < protocol["waves"]: + stop.wait(protocol["wave_pause_seconds"]) + after = snapshot(containers) + cpu_after = resource.getrusage(resource.RUSAGE_SELF) + finished = time.monotonic_ns() + save( + directory / "measurement.json", + { + "started_ns": started, + "finished_ns": finished, + "generator_cpu_seconds": cpu_after.ru_utime + + cpu_after.ru_stime + - cpu_before.ru_utime + - cpu_before.ru_stime, + "generator_maxrss_kib": cpu_after.ru_maxrss, + "idle_before": idle_before, + "idle_after": idle_after, + "before": before, + "after": after, + }, + ) + sampler.close() + outcome["sampler_errors"] = sampler.errors + sampler = None + # Downloads must not contaminate measured API network counters. + outcome["stage"] = "verify_originals" + with (directory / "verification.jsonl").open("w") as output: + for request in requests: + if stop.is_set(): + break + if not request["ready"]: + continue + try: + verified = client.verify( + actors[1], request["attachment"], len(data), fixture["sha256"] + ) + except Exception as error: + verified = { + "id": request["attachment"]["id"], + "verified": False, + "error": error.code + if isinstance(error, FlowError) + else type(error).__name__, + } + output.write(json.dumps(verified) + "\n") + output.flush() + outcome.update( + status="interrupted" if stop.is_set() else "completed", stage="done" + ) + except Exception as error: + outcome["error_type"] = type(error).__name__ + if outcome["stage"] != "startup": + outcome["status"] = "instrumentation_or_setup_failure" + finally: + if sampler: + sampler.close() + outcome["sampler_errors"] = sampler.errors + if containers: + try: + save( + directory / "containers-after.json", + inspect_containers([c["id"] for c in containers], project), + ) + except Exception as error: + outcome["inspect_error"] = type(error).__name__ + save(directory / "host-after.json", host_activity()) + save(directory / "outcome.json", outcome) + return outcome + + +def remove_runtime(runtime): + resolved = runtime.resolve() + if resolved.parent != Path("/tmp") or not resolved.name.startswith( + "highload-upload-exp-" + ): + raise ExperimentError("refusing to remove unexpected private directory") + shutil.rmtree(resolved) + + +def run_campaign(campaign, resume=False, keep_volumes=False): + check_seal(campaign) + check_sources(campaign) + protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json") + check_ports(protocol) + save(campaign / "host-run.json", host_info()) + if (campaign / "cases").exists() and not resume: + raise ExperimentError( + "campaign already started; use --resume (does not overwrite attempts)" + ) + if (campaign / "run-options.json").exists(): + if load(campaign / "run-options.json")["keep_volumes"] != keep_volumes: + raise ExperimentError( + "resume must keep the original --keep-volumes setting; cleanup separately" + ) + else: + save(campaign / "run-options.json", {"keep_volumes": keep_volumes}) + if (campaign / "runtime.json").exists(): + runtime = Path(load(campaign / "runtime.json")["private_directory"]) + if not runtime.exists(): + runtime = runtime_files(campaign) + else: + runtime = runtime_files(campaign) + stop = threading.Event() + previous = {} + for sig in (signal.SIGINT, signal.SIGTERM): + previous[sig] = signal.signal(sig, lambda *_: stop.set()) + outcomes, cleanup_failed = [], False + invocation = ( + datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") + + "-" + + secrets.token_hex(3) + ) + try: + for case in plan: + if stop.is_set(): + break + directory = campaign / "cases" / case["directory"] + if directory.exists(): + print("Preserved existing attempt: " + case["directory"], flush=True) + continue + check_sources(campaign) + check_seal(campaign) + project = project_name(campaign, case["number"]) + # Refuse reusing a project even after an earlier hard interruption. + if command( + [ + "docker", + "volume", + "ls", + "-q", + "--filter", + "label=com.docker.compose.project=" + project, + ] + ): + raise ExperimentError( + "existing experiment volumes found; cleanup or prepare a new campaign" + ) + print("Running " + case["directory"], flush=True) + try: + outcome = run_case(campaign, case, runtime, project, stop) + outcomes.append({"case": case["directory"], **outcome}) + finally: + try: + with (directory / "shutdown.log").open("w") as log: + safe_down(campaign, runtime, project, log, keep_volumes) + except Exception as error: + cleanup_failed = True + save( + directory / "cleanup-failure.json", + {"error_type": type(error).__name__, "project": project}, + ) + raise ExperimentError( + "cleanup failed; inspect evidence and use cleanup command" + ) from None + execution = {"interrupted": stop.is_set(), "last_invocation": outcomes} + save(campaign / "invocations" / (invocation + ".json"), execution) + save(campaign / "execution.json", execution) + check_sources(campaign) + print( + "Uploads finished; raw evidence retained. Run summarize; no human conclusion has been written.", + flush=True, + ) + except Exception as error: + save( + campaign / "execution-failure.json", + { + "error_type": type(error).__name__, + "reason": str(error) + if isinstance(error, ExperimentError) + else "execution interrupted by instrumentation/environment failure", + }, + ) + raise + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + if not keep_volumes and not cleanup_failed: + remove_runtime(runtime) + else: + print( + "Private runtime retained outside evidence: " + str(runtime), flush=True + ) + return 130 if stop.is_set() else 0 + + +def cleanup_campaign(campaign): + runtime = Path(load(campaign / "runtime.json")["private_directory"]) + if not runtime.exists(): + raise ExperimentError( + "runtime already removed; normal run already removed its containers/volumes" + ) + for case_file in sorted((campaign / "cases").glob("*/case.json")): + case = load(case_file) + with (case_file.parent / "manual-cleanup.log").open("a") as log: + safe_down(campaign, runtime, case["project"], log, False) + remove_runtime(runtime) + print("Only selected campaign projects removed; evidence kept.") + + +def main(): + parser = argparse.ArgumentParser( + description="Reproducible proxy/presigned upload experiment (WSL/Linux, Python stdlib)" + ) + sub = parser.add_subparsers(dest="action", required=True) + sub.add_parser("check", help="read-only prerequisites; no build or workload") + p = sub.add_parser( + "prepare", help="freeze protocol/source/fixtures and build images; no uploads" + ) + p.add_argument("--pilot", action="store_true") + p.add_argument("--reuse-images", type=campaign_path) + for action in ("run", "summarize", "cleanup"): + p = sub.add_parser(action) + p.add_argument("campaign", type=campaign_path) + if action == "run": + p.add_argument("--resume", action="store_true") + p.add_argument("--keep-volumes", action="store_true") + args = parser.parse_args() + EVIDENCE.mkdir(parents=True, exist_ok=True) + with (EVIDENCE / ".lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + raise SystemExit( + "Another experiment command is running; do not run campaigns concurrently." + ) + try: + if args.action == "check": + info = preflight() + print(json.dumps(info, ensure_ascii=False, indent=2)) + print( + "Prerequisites readable, experiment ports free; no uploads executed." + ) + elif args.action == "prepare": + prepare(args.pilot, args.reuse_images) + elif args.action == "run": + raise SystemExit( + run_campaign(args.campaign, args.resume, args.keep_volumes) + ) + elif args.action == "cleanup": + cleanup_campaign(args.campaign) + else: + check_seal(args.campaign) + result = summarize(args.campaign) + print(json.dumps(result["decision"], ensure_ascii=False, indent=2)) + except ( + ExperimentError, + OSError, + subprocess.TimeoutExpired, + ValueError, + KeyError, + ) as error: + print( + "ERROR: " + + ( + str(error) + if isinstance(error, ExperimentError) + else type(error).__name__ + ), + file=sys.stderr, + ) + raise SystemExit(1) diff --git a/experiments/src/fixtures.py b/experiments/src/fixtures.py new file mode 100644 index 00000000..919b1f97 --- /dev/null +++ b/experiments/src/fixtures.py @@ -0,0 +1,35 @@ +import hashlib +import struct +import zlib + + +def chunk(kind, payload): + return ( + struct.pack(">I", len(payload)) + + kind + + payload + + struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF) + ) + + +def make_png(width, height, size, seed): + compressor = zlib.compressobj(6) + compressed = [] + for y in range(height): + color = bytes(((seed + y) % 256, (seed + y * 3) % 256, (seed + y * 7) % 256)) + compressed.append(compressor.compress(b"\0" + color * width)) + compressed.append(compressor.flush()) + header = b"\x89PNG\r\n\x1a\n" + chunk( + b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + ) + image = header + chunk(b"IDAT", b"".join(compressed)) + # npAD: ancillary, private, reserved uppercase, unsafe-to-copy. The same + # decoded pixels occur at both encoded sizes; only this chunk grows. + remaining = size - len(image) - len(chunk(b"IEND", b"")) - 12 + if remaining < 0: + raise ValueError("target size cannot hold this PNG") + block = hashlib.sha256(str(seed).encode("ascii")).digest() + padding = (block * ((remaining + len(block) - 1) // len(block)))[:remaining] + result = image + chunk(b"npAD", padding) + chunk(b"IEND", b"") + assert len(result) == size + return result diff --git a/experiments/src/telemetry.py b/experiments/src/telemetry.py new file mode 100644 index 00000000..26ae1eed --- /dev/null +++ b/experiments/src/telemetry.py @@ -0,0 +1,171 @@ +import json +import subprocess +import threading +import time + +from common import ExperimentError, command + + +def parse_snapshot(text): + sections, current = {}, None + for line in text.splitlines(): + if line.startswith("@"): + current = line[1:] + sections[current] = [] + elif current: + sections[current].append(line) + metrics = { + name: int(sections[name][0]) + for name in ("memory.current", "memory.peak", "memory.max") + } + metrics["memory.events"] = { + row.split()[0]: int(row.split()[1]) for row in sections["memory.events"] + } + metrics["cpu.stat"] = { + row.split()[0]: int(row.split()[1]) for row in sections["cpu.stat"] + } + interfaces = {} + for row in sections["net"]: + if ":" not in row: + continue + name, values = row.split(":", 1) + if name.strip() != "lo": + values = values.split() + interfaces[name.strip()] = { + "rx_bytes": int(values[0]), + "tx_bytes": int(values[8]), + } + if not interfaces: + raise ExperimentError("no non-loopback interfaces") + metrics["interfaces"] = interfaces + metrics["rx_bytes"] = sum(i["rx_bytes"] for i in interfaces.values()) + metrics["tx_bytes"] = sum(i["tx_bytes"] for i in interfaces.values()) + return metrics + + +def snapshot(containers): + result = {} + script = "for item in memory.current memory.peak memory.max memory.events cpu.stat; do printf '@%s\\n' \"$item\"; cat /sys/fs/cgroup/$item || exit 2; done; printf '@net\\n'; cat /proc/net/dev" + for item in containers: + start = time.monotonic_ns() + text = command(["docker", "exec", item["id"], "sh", "-c", script], timeout=15) + result[item["id"]] = { + **parse_snapshot(text), + "read_started_ns": start, + "read_finished_ns": time.monotonic_ns(), + } + return result + + +def inspect_containers(ids, project): + result = [] + for item in json.loads(command(["docker", "inspect", *ids])): + labels = item["Config"]["Labels"] + if labels.get("com.docker.compose.project") != project: + raise ExperimentError("container belongs to a different project") + result.append( + { + "id": item["Id"], + "hostname": item["Config"]["Hostname"], + "service": labels["com.docker.compose.service"], + "image": item["Image"], + "started_at": item["State"]["StartedAt"], + "status": item["State"]["Status"], + "oom_killed": item["State"]["OOMKilled"], + "restarts": item["RestartCount"], + "memory_limit": item["HostConfig"]["Memory"], + "nano_cpus": item["HostConfig"]["NanoCpus"], + "memory_and_swap_limit": item["HostConfig"]["MemorySwap"], + "tmpfs": item["HostConfig"].get("Tmpfs", {}), + "volumes": [ + m.get("Name") for m in item["Mounts"] if m["Type"] == "volume" + ], + } + ) + return result + + +class Sampler: + def __init__(self, containers, path, interval): + self.file = path.open("w", encoding="utf-8") + self.lock = threading.Lock() + self.items, self.threads, self.errors = [], [], [] + self.counts = {c["id"]: 0 for c in containers} + self.stopping = False + # The in-container stop marker prevents docker-exec client termination + # from leaving a sampling shell running inside a retained container. + marker = "/tmp/upload-experiment-sampler.stop" + script = 'while [ ! -e "$1" ]; do read current < /sys/fs/cgroup/memory.current || exit 2; read peak < /sys/fs/cgroup/memory.peak || exit 2; read uptime rest < /proc/uptime; printf "%s %s %s\\n" "$current" "$peak" "$uptime"; sleep "$2"; done' + for c in containers: + process = subprocess.Popen( + [ + "docker", + "exec", + c["id"], + "sh", + "-c", + script, + "sampler", + marker, + str(interval), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.items.append((c, process, marker)) + thread = threading.Thread(target=self.read, args=(c, process), daemon=True) + thread.start() + self.threads.append(thread) + + def read(self, container, process): + try: + for line in process.stdout: + current, peak, uptime = line.split() + row = { + "container": container["id"], + "service": container["service"], + "monotonic_ns": time.monotonic_ns(), + "source_uptime_seconds": float(uptime), + "memory_current": int(current), + "memory_peak_lifetime": int(peak), + } + with self.lock: + self.counts[container["id"]] += 1 + self.file.write(json.dumps(row) + "\n") + self.file.flush() + if not self.stopping: + self.errors.append("sampler exited before stop: " + container["id"]) + except Exception as error: + self.errors.append(type(error).__name__) + + def wait_ready(self): + deadline = time.monotonic() + 10 + while not all(count >= 2 for count in self.counts.values()): + if self.errors or time.monotonic() > deadline: + raise ExperimentError("sampler did not become ready") + time.sleep(0.05) + + def close(self): + self.stopping = True + for c, process, marker in self.items: + try: + command(["docker", "exec", c["id"], "touch", marker], timeout=3) + except Exception: + self.errors.append("cannot stop sampler: " + c["id"]) + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + if process.returncode: + self.errors.append("sampler exit " + str(process.returncode)) + for thread in self.threads: + thread.join(timeout=3) + if thread.is_alive(): + self.errors.append("sampler reader did not stop") + self.file.close() diff --git a/experiments/tests/test_instruments.py b/experiments/tests/test_instruments.py new file mode 100644 index 00000000..8a8d0814 --- /dev/null +++ b/experiments/tests/test_instruments.py @@ -0,0 +1,416 @@ +import copy +import hashlib +import json +import struct +import sys +import tempfile +import threading +import unittest +import zlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +from common import EXPERIMENT, EVIDENCE, ROOT, save, load, ExperimentError +from driver import make_plan, safe_down, validate_compose +from fixtures import make_png +from telemetry import parse_snapshot +from analysis import percentile, decision, case_metrics +from client import Client + + +PROTOCOL = load(EXPERIMENT / "config/protocol.json") + + +class InstrumentTests(unittest.TestCase): + def test_evidence_is_in_repository_root(self): + self.assertEqual(EVIDENCE, ROOT / "evidence" / "upload-comparison") + + def test_png_is_valid_deterministic_and_exact_size(self): + for width, height, size in [(16, 8, 4096), (32, 16, 8192)]: + data = make_png(width, height, size, 123) + self.assertEqual(data, make_png(width, height, size, 123)) + self.assertEqual(len(data), size) + self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n") + chunks, offset = {}, 8 + while offset < len(data): + length = struct.unpack(">I", data[offset : offset + 4])[0] + kind, payload = ( + data[offset + 4 : offset + 8], + data[offset + 8 : offset + 8 + length], + ) + crc = struct.unpack( + ">I", data[offset + 8 + length : offset + 12 + length] + )[0] + self.assertEqual(crc, zlib.crc32(kind + payload) & 0xFFFFFFFF) + chunks[kind] = payload + offset += length + 12 + self.assertEqual(offset, len(data)) + self.assertEqual(struct.unpack(">II", chunks[b"IHDR"][:8]), (width, height)) + self.assertEqual( + len(zlib.decompress(chunks[b"IDAT"])), height * (width * 3 + 1) + ) + self.assertIn(b"npAD", chunks) + + def test_all_actual_profile_headers_and_lengths(self): + for profile in PROTOCOL["profiles"]: + data = make_png( + profile["width"], + profile["height"], + profile["size_bytes"], + PROTOCOL["seed"], + ) + self.assertEqual(len(data), profile["size_bytes"]) + self.assertEqual( + struct.unpack(">II", data[16:24]), (profile["width"], profile["height"]) + ) + + def test_plan_has_complete_counterbalanced_pairs(self): + plan = make_plan(PROTOCOL) + self.assertEqual(len(plan), 24) + self.assertEqual(plan, make_plan(PROTOCOL)) + self.assertEqual(sum(p["mode"] == "proxy" for p in plan[::2]), 6) + self.assertEqual(len({p["directory"] for p in plan}), 24) + for profile in PROTOCOL["profiles"]: + for repetition in range(1, 4): + pair = [ + c["mode"] + for c in plan + if c["profile"] == profile["name"] and c["repetition"] == repetition + ] + self.assertEqual(sorted(pair), ["direct", "proxy"]) + changed = copy.deepcopy(PROTOCOL) + changed["seed"] += 1 + self.assertNotEqual(plan, make_plan(changed)) + + def test_counters_sum_all_non_loopback_interfaces(self): + text = """@memory.current +100 +@memory.peak +200 +@memory.max +536870912 +@memory.events +oom 0 +oom_kill 0 +max 0 +@cpu.stat +usage_usec 500 +nr_throttled 1 +throttled_usec 12 +@net +Inter-| Receive | Transmit + lo: 900 0 0 0 0 0 0 0 800 0 0 0 0 0 0 0 + eth0: 100 0 0 0 0 0 0 0 200 0 0 0 0 0 0 0 + eth1: 300 0 0 0 0 0 0 0 400 0 0 0 0 0 0 0 +""" + parsed = parse_snapshot(text) + self.assertEqual(parsed["rx_bytes"], 400) + self.assertEqual(parsed["tx_bytes"], 600) + self.assertEqual(parsed["cpu.stat"]["usage_usec"], 500) + + def test_cleanup_cannot_target_another_campaign(self): + with tempfile.TemporaryDirectory() as folder: + campaign = Path(folder) / "series-formal-123" + campaign.mkdir() + save(campaign / "plan.json", make_plan(PROTOCOL)) + with patch("driver.command") as command: + with self.assertRaises(ExperimentError): + safe_down( + campaign, + Path(folder), + "hl-upload-exp-another-series-1", + None, + False, + ) + command.assert_not_called() + + def test_compose_memory_values_may_be_strings(self): + services = {} + r = PROTOCOL["resources"] + for role in ("api", "db", "redis", "s3", "haproxy"): + services[role] = { + "cpus": r[role + "_cpus"], + "mem_limit": str(r[role + "_memory_bytes"]), + "memswap_limit": str(r[role + "_memory_bytes"]), + } + services["api"].update( + deploy={"replicas": 2}, + tmpfs=["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"], + environment={ + "GOMAXPROCS": "1", + "GOMEMLIMIT": "384MiB", + "DATABASE_MAX_CONNS": "10", + }, + ) + response = SimpleNamespace( + returncode=0, stdout=json.dumps({"services": services}) + ) + with patch("driver.subprocess.run", return_value=response): + validate_compose(PROTOCOL) + services["api"]["memswap_limit"] = "1073741824" + response.stdout = json.dumps({"services": services}) + with patch("driver.subprocess.run", return_value=response): + with self.assertRaises(ExperimentError): + validate_compose(PROTOCOL) + + +class DecisionTests(unittest.TestCase): + def setUp(self): + self.plan = make_plan(PROTOCOL) + self.cases = [ + { + **c, + "api_bytes_per_ready": 1100000 if c["mode"] == "direct" else 2000000, + "system_failures": [], + "instrumentation_issues": [], + } + for c in self.plan + ] + + def result(self, protocol=None): + return decision(protocol or PROTOCOL, self.cases, self.plan) + + def test_declared_threshold_can_confirm(self): + self.assertEqual( + self.result()["criterion_status"], "confirmed_in_tested_conditions" + ) + + def test_bad_ratio_is_rejected(self): + for case in self.cases: + if case["mode"] == "direct": + case["api_bytes_per_ready"] = 1600000 + self.assertEqual(self.result()["criterion_status"], "rejected") + + def test_one_bad_pair_cannot_hide_behind_median(self): + next(c for c in self.cases if c["mode"] == "direct")["api_bytes_per_ready"] = ( + 1420000 + ) + self.assertEqual(self.result()["criterion_status"], "rejected") + + def test_oom_is_not_discarded_when_sampler_also_failed(self): + self.cases[0]["system_failures"] = ["oom_event:api"] + self.cases[0]["instrumentation_issues"] = ["sampler_error"] + self.assertEqual(self.result()["criterion_status"], "rejected") + + def test_broken_instruments_do_not_create_false_traffic_rejection(self): + direct = next(c for c in self.cases if c["mode"] == "direct") + direct["api_bytes_per_ready"] = 9000000 + direct["instrumentation_issues"] = ["network_counter_reset:api"] + self.assertEqual(self.result()["criterion_status"], "inconclusive") + + def test_missing_case_is_inconclusive(self): + self.cases.pop() + self.assertEqual(self.result()["criterion_status"], "inconclusive") + + def test_pilot_never_confirms_the_hypothesis(self): + protocol = copy.deepcopy(PROTOCOL) + protocol["purpose"] = "pilot" + self.assertEqual( + self.result(protocol)["criterion_status"], "pilot_not_for_hypothesis" + ) + + def test_percentiles_use_nearest_rank(self): + self.assertEqual(percentile(list(range(1, 31)), 0.99), 30) + self.assertEqual(percentile(list(range(1, 31)), 0.50), 15) + self.assertIsNone(percentile([], 0.50)) + + def test_user_interruption_is_not_a_server_failure(self): + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) + save(path / "case.json", {**self.plan[0]}) + save(path / "outcome.json", {"status": "interrupted"}) + (path / "requests.jsonl").write_text( + json.dumps({"ready": False, "error": "interrupted"}) + "\n" + ) + result = case_metrics(path, PROTOCOL) + self.assertEqual(result["errors"], 0) + self.assertEqual(result["cancelled"], 1) + self.assertEqual(result["system_failures"], []) + + def test_metrics_keep_window_ram_distinct_and_exclude_s3_from_api_traffic(self): + protocol = copy.deepcopy(PROTOCOL) + protocol.update(clients=2, waves=1, sample_interval_seconds=0.1) + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) + save(path / "case.json", self.plan[0]) + save(path / "outcome.json", {"status": "completed"}) + containers = [ + { + "id": cid, + "hostname": cid, + "service": role, + "status": "running", + "started_at": "same", + "restarts": 0, + "oom_killed": False, + "nano_cpus": cpu, + } + for cid, role, cpu in [ + ("a", "api", 500000000), + ("b", "api", 500000000), + ("s", "s3", 1000000000), + ] + ] + save(path / "containers-before.json", containers) + save(path / "containers-after.json", containers) + + def counters(received, sent, ns, peak): + return { + "rx_bytes": received, + "tx_bytes": sent, + "memory.current": 100, + "memory.peak": peak, + "memory.max": 536870912, + "memory.events": {"oom": 0, "oom_kill": 0, "max": 0}, + "cpu.stat": {"usage_usec": ns // 10000}, + "read_started_ns": ns, + "read_finished_ns": ns + 1, + } + + ids = [c["id"] for c in containers] + m = { + "started_ns": 10000000000, + "finished_ns": 12000000000, + "generator_cpu_seconds": 0.1, + "idle_before": {cid: counters(0, 0, 0, 300) for cid in ids}, + "idle_after": {cid: counters(0, 0, 5000000000, 300) for cid in ids}, + "before": {cid: counters(0, 0, 10000000000, 300) for cid in ids}, + "after": { + cid: counters(4000000, 4000000, 12000000000, 400) for cid in ids + }, + } + save(path / "measurement.json", m) + requests = [ + { + "wave": 0, + "ready": True, + "started_ns": 10100000000 + i * 1000000, + "finished_ns": 11900000000, + "attachment": {"id": cid, "instance": cid}, + } + for i, cid in enumerate(("a", "b")) + ] + (path / "requests.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in requests) + ) + (path / "warmup.jsonl").write_text( + "".join(json.dumps(r) + "\n" for r in requests) + ) + (path / "verification.jsonl").write_text( + "".join( + json.dumps({"id": cid, "verified": True}) + "\n" + for cid in ("a", "b") + ) + ) + (path / "samples.jsonl").write_text( + "".join( + json.dumps( + { + "container": cid, + "memory_current": 200, + "monotonic_ns": 10000000000 + i * 100000000, + } + ) + + "\n" + for cid in ids + for i in range(21) + ) + ) + result = case_metrics(path, protocol) + self.assertEqual(result["api_rx_tx_bytes"], 16000000) + self.assertEqual(result["api_bytes_per_ready"], 8000000) + self.assertEqual(result["containers"][0]["sampled_peak_window"], 200) + self.assertEqual(result["containers"][0]["peak_lifetime_after"], 400) + self.assertEqual(result["system_failures"], []) + self.assertEqual(result["instrumentation_issues"], []) + + +class TransportTests(unittest.TestCase): + def test_direct_stages_do_not_send_bearer_to_s3_or_leak_secrets_to_records(self): + observed, data = [], b"synthetic PNG transport fixture" + checksum = hashlib.sha256(data).hexdigest() + token, signed_field = "DO-NOT-PERSIST-TOKEN", "DO-NOT-PERSIST-SIGNATURE" + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + observed.append( + { + "path": self.path, + "authorization": self.headers.get("Authorization"), + "body": body, + } + ) + if self.path.endswith("/attachment-uploads"): + response = { + "upload_id": "upload", + "upload": { + "url": f"http://127.0.0.1:{self.server.server_port}/bucket", + "fields": { + "key": "staging/direct/upload", + "policy": signed_field, + }, + }, + } + elif self.path == "/bucket": + self.send_response(204) + self.send_header("Content-Length", "0") + self.end_headers() + return + else: + response = { + "id": "attachment", + "status": "ready", + "size_bytes": len(data), + "sha256": checksum, + } + encoded = json.dumps(response).encode() + self.send_response(201) + self.send_header("Content-Length", str(len(encoded))) + self.send_header("X-Instance-Id", "api-test") + self.end_headers() + self.wfile.write(encoded) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + protocol = copy.deepcopy(PROTOCOL) + protocol["ports"] = {"api": server.server_port, "s3": server.server_port} + client = Client( + f"http://127.0.0.1:{server.server_port}", protocol, threading.Event() + ) + records = [] + client.upload( + "direct", + {"token": token}, + "chat", + data, + checksum, + records, + __import__("time").monotonic() + 10, + ) + self.assertEqual( + [r["stage"] for r in records], + ["client_sha256", "init", "s3_post", "complete"], + ) + self.assertEqual(observed[0]["authorization"], "Bearer " + token) + self.assertIsNone(observed[1]["authorization"]) + self.assertIn(data, observed[1]["body"]) + self.assertEqual(observed[2]["authorization"], "Bearer " + token) + self.assertNotIn(token, json.dumps(records)) + self.assertNotIn(signed_field, json.dumps(records)) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/tests/test_orchestration.py b/experiments/tests/test_orchestration.py new file mode 100644 index 00000000..42e4697a --- /dev/null +++ b/experiments/tests/test_orchestration.py @@ -0,0 +1,213 @@ +import hashlib +import json +import sys +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +import driver +from analysis import json_lines +from client import FlowError +from common import ( + ROOT, + ExperimentError, + check_seal, + digest, + load, + save, + source_manifest, +) + + +class OrchestrationTests(unittest.TestCase): + def test_source_snapshot_excludes_secrets_evidence_and_git(self): + manifest = source_manifest() + self.assertIn("src/backend/services/direct_upload.go", manifest) + self.assertIn("experiments/docs/HYPOTHESIS.md", manifest) + for name in manifest: + self.assertNotIn( + name.split("/")[0], (".git", ".memory", "secrets", "evidence") + ) + self.assertNotEqual(Path(name).name, ".env") + + def test_prepare_seals_protocol_before_builds_and_does_not_upload(self): + with tempfile.TemporaryDirectory(dir=ROOT) as folder: + evidence = Path(folder) + names = [ + "src/go.mod", + "infra/minio.Dockerfile", + "experiments/src/analysis.py", + "experiments/src/common.py", + ] + manifest = {name: digest(ROOT / name) for name in names} + + def fake_command(args, *_args, **_kwargs): + if args[:2] == ["docker", "build"]: + campaign = next(evidence.iterdir()) + self.assertTrue((campaign / "protocol.json").exists()) + self.assertFalse((campaign / "cases").exists()) + return "" + if args[:3] == ["docker", "image", "inspect"]: + return json.dumps( + [ + { + "Id": "sha256:" + "a" * 64, + "RepoDigests": [], + "Os": "linux", + "Architecture": "amd64", + "RootFS": {"Layers": ["layer"]}, + } + ] + ) + if args[0] == "git": + return "synthetic-git-metadata" + raise AssertionError("unexpected command in prepare") + + with ( + patch("driver.EVIDENCE", evidence), + patch("driver.preflight", return_value={"purpose": "fake-host"}), + patch("driver.source_manifest", return_value=manifest), + patch("driver.check_sources"), + patch("driver.command", side_effect=fake_command), + patch("driver.Client") as client, + patch("builtins.print"), + ): + campaign = driver.prepare(pilot=True) + client.assert_not_called() + check_seal(campaign) + self.assertEqual(load(campaign / "prepared.json")["cases"], 4) + self.assertEqual(load(campaign / "protocol.json")["purpose"], "pilot") + self.assertFalse((campaign / "cases").exists()) + protocol = load(campaign / "protocol.json") + protocol["clients"] += 1 + save(campaign / "protocol.json", protocol) + with self.assertRaises(ExperimentError): + check_seal(campaign) + + def test_failed_uploads_are_saved_and_verification_starts_after_counters_close( + self, + ): + with tempfile.TemporaryDirectory() as folder: + campaign = Path(folder) + protocol = load(driver.EXPERIMENT / "config/protocol.json") + protocol.update( + clients=2, + waves=1, + warmup_clients=2, + settle_seconds=0, + idle_seconds=0.001, + ) + save(campaign / "protocol.json", protocol) + save(campaign / "images.json", {"app": {"id": "app"}, "s3": {"id": "s3"}}) + data = b"synthetic-fixture" + checksum = hashlib.sha256(data).hexdigest() + save( + campaign / "fixtures.json", + [{"name": "fake", "filename": "fake.png", "sha256": checksum}], + ) + (campaign / "fixtures").mkdir() + (campaign / "fixtures/fake.png").write_bytes(data) + case = { + "directory": "01-fake", + "number": 1, + "profile": "fake", + "mode": "direct", + "size_bytes": len(data), + "repetition": 1, + } + containers = [ + { + "id": cid, + "hostname": cid, + "service": role, + "image": image, + "memory_limit": 536870912, + "memory_and_swap_limit": 536870912, + "nano_cpus": 500000000, + "tmpfs": {"/tmp/uploads": "rw,noexec,nosuid,size=16m,mode=1777"}, + } + for cid, role, image in [ + ("a", "api", "app"), + ("b", "api", "app"), + ("s", "s3", "s3"), + ] + ] + state = {"snapshots": 0, "sampler_closed": False, "uploads": 0} + mutex = threading.Lock() + + class FakeClient: + def __init__(self, _base, p, _stop): + self.protocol = p + + def bootstrap(self): + return [{"token": "secret-a"}, {"token": "secret-b"}], "chat" + + def upload( + self, _mode, _actor, _chat, _data, _digest, records, _deadline + ): + with mutex: + state["uploads"] += 1 + index = state["uploads"] + if index == 3: + raise FlowError("http_503") + return { + "id": "attachment-" + str(index), + "instance": "a" if index % 2 else "b", + } + + def verify(self, _actor, attachment, _size, _digest): + if state["snapshots"] != 4 or not state["sampler_closed"]: + raise AssertionError("verification contaminated measurement") + return {"id": attachment["id"], "verified": True} + + class FakeSampler: + errors = [] + + def __init__(self, _containers, path, _interval): + path.write_text("") + + def wait_ready(self): + pass + + def close(self): + state["sampler_closed"] = True + + def fake_snapshot(_containers): + state["snapshots"] += 1 + return {c["id"]: {"sample": "fake"} for c in containers} + + def fake_command(args, *_args, **_kwargs): + return "a\nb\ns" if "ps" in args else "" + + with ( + patch("driver.command", side_effect=fake_command), + patch("driver.inspect_containers", return_value=containers), + patch("driver.Client", FakeClient), + patch("driver.Sampler", FakeSampler), + patch("driver.snapshot", side_effect=fake_snapshot), + ): + outcome = driver.run_case( + campaign, + case, + Path(folder), + "hl-upload-exp-fake-1", + threading.Event(), + ) + self.assertEqual(outcome["status"], "completed") + directory = campaign / "cases/01-fake" + requests = json_lines(directory / "requests.jsonl") + self.assertEqual(len(requests), 2) + self.assertEqual(sum(r["ready"] for r in requests), 1) + self.assertEqual( + next(r for r in requests if not r["ready"])["error"], "http_503" + ) + self.assertEqual(len(json_lines(directory / "verification.jsonl")), 1) + self.assertNotIn("secret-a", (directory / "requests.jsonl").read_text()) + self.assertTrue((directory / "measurement.json").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/experiments/tests/test_plots.py b/experiments/tests/test_plots.py new file mode 100644 index 00000000..e7470942 --- /dev/null +++ b/experiments/tests/test_plots.py @@ -0,0 +1,49 @@ +"""Offline plot checks; no application uploads or research measurements.""" +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +AVAILABLE = all(importlib.util.find_spec(name) for name in ("pandas", "matplotlib")) + + +@unittest.skipUnless(AVAILABLE, "plot dependencies: run bash experiments/setup.sh") +class PlotTests(unittest.TestCase): + def test_plots_preserve_inputs_and_keep_all_attempts(self): + experiment = Path(__file__).resolve().parents[1] + spec = importlib.util.spec_from_file_location("experiment_plot", experiment / "plot.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + with tempfile.TemporaryDirectory() as temporary: + campaign = Path(temporary) + protocol = {"profiles": [{"name": "sample"}], "criteria": { + "median_direct_proxy_traffic_ratio_max": 0.65, + "each_pair_traffic_ratio_max": 0.70}} + summary = {"decision": {"criterion_status": "inconclusive", "pairs": [ + {"profile": "sample", "repetition": 1, "direct_proxy_ratio": 0.6, + "instrumentation_valid": False}]}} + (campaign / "protocol.json").write_text(json.dumps(protocol)) + (campaign / "metrics.json").write_text(json.dumps(summary)) + (campaign / "metrics.csv").write_text( + "profile,mode,repetition,errors,api_bytes_per_ready," + "api_sampled_peak_window_max,api_peak_lifetime_max,p95_seconds\n" + "sample,proxy,1,1,2000000,200000000,300000000,2\n" + "sample,direct,1,0,1200000,210000000,310000000,3\n" + "sample,direct,2,2,,,,\n") + before = {f.name: f.read_bytes() for f in campaign.glob("*.json")} + before["metrics.csv"] = (campaign / "metrics.csv").read_bytes() + output = module.create_plots(campaign) + self.assertEqual(len(list(output.glob("*.png"))), 6) + self.assertEqual(len(list(output.glob("*.pdf"))), 0) + self.assertTrue(all(f.stat().st_size > 1000 for f in output.glob("*.png"))) + provenance = json.loads((output / "provenance.json").read_text()) + self.assertEqual(provenance["criterion_status"], "inconclusive") + for name, content in before.items(): + self.assertEqual((campaign / name).read_bytes(), content) + self.assertNotEqual(output, module.create_plots(campaign)) + + +if __name__ == "__main__": + unittest.main()