feat: plan the experiment
This commit is contained in:
@@ -0,0 +1,23 @@
|
||||
# Эксперимент: proxy upload и presigned POST
|
||||
|
||||
Сравниваем трафик API, память и время загрузки. Результаты всех серий — в корневом [evidence/upload-comparison](../evidence/upload-comparison/).
|
||||
|
||||
## Документация
|
||||
|
||||
- [Гипотеза и критерии](docs/HYPOTHESIS.md)
|
||||
- [Методика и метрики](docs/METHODOLOGY.md)
|
||||
- [Ограничения](docs/LIMITATIONS.md)
|
||||
- [Запуск, графики и повтор](docs/RUNBOOK.md)
|
||||
|
||||
## Структура
|
||||
|
||||
| Каталог / файл | Назначение |
|
||||
| --- | --- |
|
||||
| [config/protocol.json](config/protocol.json) | Параметры и пороги эксперимента |
|
||||
| [config/compose.yml](config/compose.yml) | Отдельный стенд |
|
||||
| [src](src/) | Генератор данных, нагрузка и измерители |
|
||||
| [tests](tests/) | Проверки инструментов |
|
||||
| [plot.py](plot.py) | Графики из результатов серии |
|
||||
| [setup.sh](setup.sh) | Создание корневого `.venv` для анализа |
|
||||
|
||||
Начни с [инструкции запуска](docs/RUNBOOK.md): проверка окружения → пилот → основная серия → графики. Пилот проверяет инструменты и не подтверждает гипотезу. Измерения пока не выполнены.
|
||||
@@ -0,0 +1,157 @@
|
||||
services:
|
||||
db:
|
||||
image: ${DB_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
memswap_limit: 512m
|
||||
networks: [data]
|
||||
environment:
|
||||
POSTGRES_DB: messenger_exp
|
||||
POSTGRES_USER: messenger_exp
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
- ${SECRET_DIR}/db_password:/run/secrets/db_password:ro
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, pg_isready -U messenger_exp -d messenger_exp]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
redis:
|
||||
image: ${REDIS_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks: [data]
|
||||
command: [redis-server, --appendonly, "yes", --appendfsync, everysec, --dir, /data]
|
||||
volumes: [redis_data:/data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
s3:
|
||||
image: ${S3_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 1
|
||||
mem_limit: 512m
|
||||
memswap_limit: 512m
|
||||
networks: [edge, data]
|
||||
ports: ["127.0.0.1:${S3_PORT}:9000"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/root_access
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/root_secret
|
||||
MINIO_BROWSER: "off"
|
||||
MINIO_API_CORS_ALLOW_ORIGIN: http://127.0.0.1:5173,http://localhost:5173
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ${SECRET_DIR}/root_access:/run/secrets/root_access:ro
|
||||
- ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro
|
||||
healthcheck:
|
||||
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/ready]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
migrate:
|
||||
image: ${APP_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
volumes:
|
||||
- ${SECRET_DIR}/database_url:/run/secrets/database_url:ro
|
||||
command: [./messenger, migrate, up]
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
s3-bootstrap:
|
||||
image: ${APP_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
environment:
|
||||
<<: &s3
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_PUBLIC_ENDPOINT: http://127.0.0.1:${S3_PORT}
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-experiment
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/api_access
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/api_secret
|
||||
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/root_access
|
||||
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/root_secret
|
||||
volumes:
|
||||
- ${SECRET_DIR}/api_access:/run/secrets/api_access:ro
|
||||
- ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro
|
||||
- ${SECRET_DIR}/root_access:/run/secrets/root_access:ro
|
||||
- ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro
|
||||
command: [./messenger, s3-bootstrap]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
api:
|
||||
image: ${APP_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 0.5
|
||||
mem_limit: 512m
|
||||
memswap_limit: 512m
|
||||
deploy: {replicas: 2}
|
||||
networks: [proxy, data]
|
||||
stop_grace_period: 15s
|
||||
environment:
|
||||
<<: *s3
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: experiment
|
||||
GOMAXPROCS: "1"
|
||||
GOMEMLIMIT: 384MiB
|
||||
DATABASE_MAX_CONNS: "10"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
IDEMPOTENCY_TTL_SECONDS: "86400"
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
UPLOAD_CLEANUP_GRACE_SECONDS: "600"
|
||||
volumes:
|
||||
- ${SECRET_DIR}/api_access:/run/secrets/api_access:ro
|
||||
- ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro
|
||||
- ${SECRET_DIR}/database_url:/run/secrets/database_url:ro
|
||||
- ${SECRET_DIR}/cache_url:/run/secrets/cache_url:ro
|
||||
tmpfs: ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-bootstrap: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, ./messenger, healthcheck]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
haproxy:
|
||||
image: ${HAPROXY_IMAGE}
|
||||
pull_policy: never
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
memswap_limit: 128m
|
||||
networks: [edge, proxy]
|
||||
environment: {API_REPLICAS: "2", APP_PORT: "8080"}
|
||||
ports: ["127.0.0.1:${API_PORT}:8080", "127.0.0.1:${STATS_PORT}:8404"]
|
||||
volumes:
|
||||
- ${REPO_ROOT}/infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
networks:
|
||||
edge:
|
||||
proxy: {internal: true}
|
||||
data: {internal: true}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"version": 1,
|
||||
"name": "proxy-versus-presigned-post",
|
||||
"purpose": "confirmatory",
|
||||
"hypothesis": "Presigned POST uses at most 65% of proxy API RX+TX bytes per ready, byte-verified attachment in each profile, without request errors, OOM or API restarts.",
|
||||
"seed": 20261018,
|
||||
"repetitions": 3,
|
||||
"clients": 10,
|
||||
"waves": 3,
|
||||
"wave_pause_seconds": 1,
|
||||
"warmup_clients": 4,
|
||||
"settle_seconds": 2,
|
||||
"idle_seconds": 5,
|
||||
"sample_interval_seconds": 0.1,
|
||||
"request_timeout_seconds": 35,
|
||||
"flow_timeout_seconds": 90,
|
||||
"ports": {"api": 18080, "s3": 19000, "stats": 18404},
|
||||
"resources": {
|
||||
"api_replicas": 2,
|
||||
"api_cpus": 0.5,
|
||||
"api_memory_bytes": 536870912,
|
||||
"api_tmpfs_bytes": 16777216,
|
||||
"api_go_memory_limit": "384MiB",
|
||||
"api_gomaxprocs": 1,
|
||||
"database_max_connections_per_replica": 10,
|
||||
"db_cpus": 1,
|
||||
"db_memory_bytes": 536870912,
|
||||
"s3_cpus": 1,
|
||||
"s3_memory_bytes": 536870912,
|
||||
"redis_cpus": 0.25,
|
||||
"redis_memory_bytes": 134217728,
|
||||
"haproxy_cpus": 0.5,
|
||||
"haproxy_memory_bytes": 134217728
|
||||
},
|
||||
"images": {"db": "postgres:16.10-bookworm", "redis": "redis:7.4.11-bookworm", "haproxy": "haproxy:3.2.25-alpine"},
|
||||
"profiles": [
|
||||
{"name": "small-moderate", "size_bytes": 1048576, "width": 1024, "height": 1024},
|
||||
{"name": "large-moderate", "size_bytes": 10485760, "width": 1024, "height": 1024},
|
||||
{"name": "small-pixels40m", "size_bytes": 1048576, "width": 8000, "height": 5000},
|
||||
{"name": "large-pixels40m", "size_bytes": 10485760, "width": 8000, "height": 5000}
|
||||
],
|
||||
"criteria": {
|
||||
"median_direct_proxy_traffic_ratio_max": 0.65,
|
||||
"each_pair_traffic_ratio_max": 0.70,
|
||||
"errors_allowed": 0,
|
||||
"api_peak_strictly_below_bytes": 536870912,
|
||||
"start_spread_seconds_max": 0.1,
|
||||
"idle_background_payload_fraction_max": 0.05,
|
||||
"sampler_coverage_fraction_min": 0.7,
|
||||
"sampler_gap_seconds_max": 1.0,
|
||||
"generator_cpu_cores_max": 0.8
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
# Гипотеза, версия 1
|
||||
|
||||
[README](../README.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md)
|
||||
|
||||
**H1:** presigned POST уменьшает суммарный RX+TX двух API на готовое, побайтно проверенное вложение минимум на 35% по медиане парных повторов каждого профиля. Ошибок, OOM и рестартов быть не должно.
|
||||
|
||||
Proxy принимает файл и передаёт его S3. Direct отправляет файл сразу в S3, но API скачивает конечный объект для проверки. Поэтому ожидается снижение трафика API; снижение RAM не обещается.
|
||||
|
||||
## Показатель и критерии
|
||||
|
||||
```text
|
||||
b = сумма RX+TX двух API за окно / число ready-вложений
|
||||
R = b_direct / b_proxy для одного профиля и повтора
|
||||
```
|
||||
|
||||
Все ready-вложения затем проверяются по длине и SHA-256. Фон измеряется отдельно и не вычитается.
|
||||
|
||||
Для подтверждения нужны все условия:
|
||||
|
||||
- По каждому из 4 профилей есть 3 полные пары: медиана R ≤ 0.65, каждый R ≤ 0.70.
|
||||
- Все 720 измеряемых загрузок и 96 загрузок прогрева успешны; оригиналы измеряемых загрузок совпали.
|
||||
- Нет OOM, рестартов и завершений API/S3; lifetime peak каждой API строго ниже 512 МиБ.
|
||||
- Обе API участвовали в прогреве и измерении.
|
||||
|
||||
## Пригодность измерений
|
||||
|
||||
| Проверка | Порог |
|
||||
| --- | --- |
|
||||
| Разброс старта клиентов в волне | ≤100 мс |
|
||||
| Покрытие RAM-точками с шагом 100 мс | ≥70% |
|
||||
| Максимальный промежуток между точками, включая границы окна | ≤1 с |
|
||||
| Оценённый фон API относительно payload прогона | ≤5% |
|
||||
| Средний CPU генератора | <0.8 ядра |
|
||||
|
||||
Нужны полные записи запросов, проверок и счётчиков без необъяснённых сбросов.
|
||||
|
||||
## Решение
|
||||
|
||||
- `confirmed_in_tested_conditions` — все критерии выполнены.
|
||||
- `rejected` — ошибки системы/оригиналов, OOM/рестарт/предел памяти либо превышение порогов трафика на пригодных данных.
|
||||
- `inconclusive` — данных недостаточно или измерители/запуск нарушены, системного опровержения нет.
|
||||
- `pilot_not_for_hypothesis` — пилот, независимо от чисел.
|
||||
|
||||
Ошибка измерителя не отменяет известный OOM или отказ приложения. Отмена пользователем не считается отказом приложения. Все попытки сохраняются. Пороги фиксируются до запуска; изменение требует новой версии протокола и новой серии.
|
||||
@@ -0,0 +1,16 @@
|
||||
# Ограничения
|
||||
|
||||
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Запуск](RUNBOOK.md)
|
||||
|
||||
- **Один WSL/Docker-host.** Сервисы и генератор делят ресурсы. Повторы не являются независимыми машинами; посторонняя нагрузка и page cache могут влиять на результат.
|
||||
- **Локальный Python-клиент, без TLS.** Новое TCP-соединение на каждой стадии. Результат не переносится напрямую на WAN, браузер и React/Web Crypto.
|
||||
- **Только синтетический RGB8 PNG.** Дополнительный chunk управляет encoded-размером. Это не модель JPEG-фотографий или всех допустимых изображений.
|
||||
- **Короткие волны нагрузки.** 10 клиентов, максимум два декодера. Эксперимент не определяет максимальный RPS или длительную устойчивость. Три пары дают ограниченную оценку разброса; p99 из 30 запросов близок к максимуму.
|
||||
- **Direct не устраняет стоимость обработки.** API читает и декодирует final; S3 делает COPY и хранит staging. Снижение трафика API не означает снижение общей стоимости системы. Дисковый I/O отдельно не исследуется.
|
||||
- **RAM-пики различаются.** Sampled max может пропустить короткий всплеск; kernel lifetime peak включает startup и прогрев. tmpfs и sampler входят в память cgroup; GOMEMLIMIT не равен всей памяти контейнера.
|
||||
- **Измерители создают нагрузку.** Sampler, хеширование и запись evidence расходуют CPU/RAM. Порог CPU генератора лишь сигнал возможного ограничения. Сетевые счётчики включают SQL, Redis и healthcheck; фон не вычитается.
|
||||
- **Повтор зависит от артефактов.** Нужны сохранённые fixtures, исходники и image ID. Пересборка после удаления образов может дать другую базу; версии хоста, Python и zlib фиксируются.
|
||||
- **Отказы не скрываются.** Все попытки сохраняются. Ошибки/OOM опровергают работоспособность; пропуски измерителя могут сделать результат неопределённым. Исправление проверяется новой серией.
|
||||
- **Без fault/security-нагрузки.** ACL, подписи и компенсация проверяются отдельно. Cleanup worker исключён из экспериментального стенда, чтобы не менять измеряемый трафик.
|
||||
|
||||
До фактических измерений и проверки качества данных выводов о выигрыше нет.
|
||||
@@ -0,0 +1,56 @@
|
||||
# Методика, версия 1
|
||||
|
||||
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md)
|
||||
|
||||
## Сравнение
|
||||
|
||||
- **Proxy:** binary POST в API до `201 ready`.
|
||||
- **Direct:** SHA-256 на клиенте → init API → presigned POST в S3 → complete API до `201 ready`.
|
||||
|
||||
Повторов запросов нет. Обе ветки используют один образ API и полную проверку изображения. Direct включает COPY внутри S3 и чтение конечного объекта API.
|
||||
|
||||
## Стенд и данные
|
||||
|
||||
| Сервис | CPU | RAM |
|
||||
| --- | --- | --- |
|
||||
| API ×2 | по 0.5 | по 512 МиБ |
|
||||
| PostgreSQL / MinIO | по 1 | по 512 МиБ |
|
||||
| Redis | 0.25 | 128 МиБ |
|
||||
| HAProxy | 0.5 | 128 МиБ |
|
||||
|
||||
Каждая API: один декодер, tmpfs 16 МиБ, GOMAXPROCS=1, GOMEMLIMIT=384MiB, пул БД=10. Swap отключён. Таймауты: операция API 25 с, HTTP 30 с, HAProxy/socket 35 с; бюджет flow 90 с проверяется между стадиями.
|
||||
|
||||
Четыре RGB8 PNG-профиля: **1 и 10 МиБ × 1024×1024 и 8000×5000 пикселей**. Размер доводится корректным ancillary chunk `npAD` перед IEND; в паре размеров одной геометрии пиксели одинаковые. Файлы и SHA-256 сохраняются.
|
||||
|
||||
## Порядок
|
||||
|
||||
Seed `20261018`, 3 парных повтора каждого профиля, порядок пар перемешан; 6 пар proxy→direct и 6 direct→proxy. Всего 24 прогона, 720 измеряемых загрузок.
|
||||
|
||||
Для каждого прогона:
|
||||
|
||||
1. Поднять отдельный Compose-проект со свежими volumes и закреплёнными image ID. Создать двух пользователей и чат.
|
||||
2. Выполнить 4 параллельных загрузки прогрева, проверить метаданные; подождать 2 с.
|
||||
3. Запустить sampler API/S3 с шагом 100 мс. Измерить фон за 5 с.
|
||||
4. Снять начальные счётчики. Выполнить 3 волны по 10 клиентов со стартовым барьером и паузой 1 с между волнами.
|
||||
5. Снять конечные счётчики и остановить sampler. Затем скачать оригиналы вторым участником и проверить длину, MIME и SHA-256.
|
||||
6. Сохранить состояния и ошибки, остановить только этот проект. По умолчанию удалить его volumes.
|
||||
|
||||
Файл заранее находится в RAM клиента; direct checksum входит во время flow. Исходники не меняются, параллельная нагрузка запрещена. Ошибки запросов сохраняются, серия продолжается; отмена, изменение исходников или сбой окружения/уборки останавливают запуск.
|
||||
|
||||
## Метрики и сохранение
|
||||
|
||||
| Метрика | Источник / смысл |
|
||||
| --- | --- |
|
||||
| API RX+TX | Разность `/proc/net/dev` всех интерфейсов кроме lo, сумма двух API; служебный обмен включён |
|
||||
| S3 RX+TX | Отдельный показатель, не складывается с API |
|
||||
| RAM в окне | Максимум sampled `memory.current`, baseline и интервалы точек |
|
||||
| Lifetime RAM | `memory.peak` до/после, включая startup и прогрев |
|
||||
| CPU / throttling / OOM | `cpu.stat`, `memory.events`, состояния и рестарты контейнеров |
|
||||
| Время / goodput | Monotonic clock; стадии, p50/p95/p99 успешных flow, ready/сек за окно |
|
||||
| Генератор | CPU, RSS, фактический разброс старта клиентов |
|
||||
|
||||
`prepare` фиксирует протокол, план, исходники, fixtures и image ID в `seal.json`. Перед прогонами проверяются контрольные суммы. `summarize` создаёт `metrics.json`, `metrics.csv` и provenance; сравнивает парные R и их медиану отдельно по профилям.
|
||||
|
||||
В корневом `evidence/upload-comparison/<серия>/`: замороженная конфигурация и исходники, `cases/*` с запросами, RAM-точками, счётчиками, проверками файлов и ошибками. Секреты и подписанные формы не записываются. Временные credentials находятся вне evidence в `/tmp/highload-upload-exp-*`.
|
||||
|
||||
Источники измерителей: [сетевая статистика Linux](https://www.kernel.org/doc/html/latest/networking/statistics.html), [cgroup v2](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html).
|
||||
@@ -0,0 +1,80 @@
|
||||
# Запуск и повтор
|
||||
|
||||
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md)
|
||||
|
||||
## Подготовка
|
||||
|
||||
Команды выполнять в **WSL/Linux из корня лабораторной**. Для закреплённого окружения анализа нужны Python 3.12, модуль `venv`, Docker с cgroup v2/`memory.peak` и Compose v2+. Ориентир: 4 CPU, 6 ГиБ RAM Docker, 10 ГиБ диска. Порты 18080, 19000, 18404 должны быть свободны.
|
||||
|
||||
```sh
|
||||
bash experiments/setup.sh
|
||||
.venv/bin/python -m unittest discover -s experiments/tests -v
|
||||
.venv/bin/python experiments/run.py check
|
||||
```
|
||||
|
||||
`setup.sh` создаёт `.venv` в корне и устанавливает pandas/matplotlib из [requirements.txt](../requirements.txt). Версии прямых и транзитивных зависимостей закреплены. Они нужны только для графиков; runner использует стандартную библиотеку (Python 3.9+). `.venv` уже исключён из Git. Если `venv` недоступен: `sudo apt install python3-venv`.
|
||||
|
||||
Не менять исходники между prepare и концом серии, не запускать параллельно сборки/тесты/нагрузку. Рабочий стенд не останавливается автоматически; оставь его без нагрузки.
|
||||
|
||||
## Пилот
|
||||
|
||||
```sh
|
||||
.venv/bin/python experiments/run.py prepare --pilot
|
||||
# Подставь путь из строки Prepared:
|
||||
PILOT=evidence/upload-comparison/<каталог-pilot>
|
||||
.venv/bin/python experiments/run.py run "$PILOT"
|
||||
.venv/bin/python experiments/run.py summarize "$PILOT"
|
||||
.venv/bin/python experiments/plot.py "$PILOT"
|
||||
```
|
||||
|
||||
Пилот: 2 moderate-профиля, 2 клиента, 1 волна, 1 парный повтор — 4 прогона и 8 измеряемых загрузок. Он проверяет инструменты и не участвует в H1. При исправлении инструментов сохрани старые данные и подготовь новую серию.
|
||||
|
||||
## Основная серия
|
||||
|
||||
```sh
|
||||
.venv/bin/python experiments/run.py prepare --reuse-images "$PILOT"
|
||||
CAMPAIGN=evidence/upload-comparison/<каталог-formal>
|
||||
.venv/bin/python experiments/run.py run "$CAMPAIGN"
|
||||
.venv/bin/python experiments/run.py summarize "$CAMPAIGN"
|
||||
.venv/bin/python experiments/plot.py "$CAMPAIGN"
|
||||
```
|
||||
|
||||
При изменении исходников или без пилота используй `prepare` без `--reuse-images`. Подготовка собирает собственные образы и фиксирует их ID, но не выполняет загрузки. Основная серия: 24 прогона, 720 измеряемых загрузок.
|
||||
|
||||
`run`: код 0 — план завершён, 1 — сбой окружения/оркестратора, 130 — отмена. Решение о H1 находится в `metrics.json → decision`, а не в коде выхода.
|
||||
|
||||
## Результаты
|
||||
|
||||
Весь каталог серии сохраняется в **корневом `evidence/upload-comparison/`**:
|
||||
|
||||
- `protocol.json`, `plan.json`, `seal.json`, исходники, fixtures и image ID — условия повторения.
|
||||
- `cases/*` — сырые запросы, RAM-точки, счётчики, проверки оригиналов, состояния и ошибки.
|
||||
- `metrics.json` / `metrics.csv` — итоговые показатели и критерии; `analysis-provenance.json` — версия анализа.
|
||||
- `plots/<время>/` — PNG-графики трафика, отношения direct/proxy, RAM, p95 и ошибок; `provenance.json` фиксирует входы и версии библиотек.
|
||||
|
||||
Графики показывают все доступные прогоны; пропуски не заменяются нулями. Качество измерений и итог H1 проверяй по JSON. Для разбора передай весь каталог, включая неудачные прогоны. `/tmp/highload-upload-exp-*` содержит credentials: его не архивировать.
|
||||
|
||||
## Повтор, остановка и уборка
|
||||
|
||||
Новый полный повтор без перезаписи:
|
||||
|
||||
```sh
|
||||
.venv/bin/python experiments/run.py prepare --reuse-images "$CAMPAIGN"
|
||||
# Подставь новый путь и повтори run → summarize → plot.
|
||||
```
|
||||
|
||||
Один `Ctrl+C` сохраняет данные и запускает уборку. Активные запросы завершаются в пределах таймаута. Продолжить только ещё не начатые прогоны:
|
||||
|
||||
```sh
|
||||
.venv/bin/python experiments/run.py run "$CAMPAIGN" --resume
|
||||
```
|
||||
|
||||
Прерванные попытки остаются в evidence и могут дать `inconclusive`; для полного повторения нужна новая серия. Не менять seal/state вручную.
|
||||
|
||||
`run ... --keep-volumes` оставляет экспериментальные volumes для отладки; при resume сохрани тот же флаг. После разбора, аварийного завершения WSL или ошибки уборки:
|
||||
|
||||
```sh
|
||||
.venv/bin/python experiments/run.py cleanup "$CAMPAIGN"
|
||||
```
|
||||
|
||||
Удаляются только ресурсы выбранной серии, evidence остаётся. При обычном завершении уборка автоматическая. Исходники и документы входят в manifest: их изменение требует нового prepare. Для summarize старой серии `analysis.py` и `common.py` должны совпадать с архивом `source.tar.gz`; старые образы сохраняй через `docker image save/load`.
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot saved summaries without changing raw evidence or the H1 decision."""
|
||||
import argparse
|
||||
import datetime
|
||||
import hashlib
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import pandas as pd
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
from common import campaign_path
|
||||
|
||||
|
||||
def sha256(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def create_plots(campaign):
|
||||
inputs = [campaign / "metrics.csv", campaign / "metrics.json"]
|
||||
frame = pd.read_csv(inputs[0])
|
||||
summary = json.loads(inputs[1].read_text())
|
||||
protocol = json.loads((campaign / "protocol.json").read_text())
|
||||
profiles = [p["name"] for p in protocol["profiles"]]
|
||||
status = summary["decision"]["criterion_status"]
|
||||
output = campaign / "plots" / datetime.datetime.now(
|
||||
datetime.timezone.utc
|
||||
).strftime("%Y%m%dT%H%M%S.%fZ")
|
||||
output.mkdir(parents=True)
|
||||
|
||||
def finish(fig, ax, title, ylabel, name):
|
||||
ax.set_title(title + "\n" + status)
|
||||
ax.set_ylabel(ylabel)
|
||||
ax.set_xticks(range(len(profiles)), profiles, rotation=20, ha="right")
|
||||
ax.grid(axis="y", alpha=0.25)
|
||||
ax.legend(loc="best", fontsize=8)
|
||||
fig.tight_layout()
|
||||
fig.savefig(output / (name + ".png"), dpi=160)
|
||||
plt.close(fig)
|
||||
|
||||
def points(column, scale, title, ylabel, name, limit=None):
|
||||
fig, ax = plt.subplots(figsize=(10, 5))
|
||||
for mode, offset, color in [("proxy", -0.12, "#2563eb"),
|
||||
("direct", 0.12, "#d97706")]:
|
||||
subset = frame[frame["mode"] == mode]
|
||||
x, y = [], []
|
||||
for index, profile in enumerate(profiles):
|
||||
values = pd.to_numeric(
|
||||
subset.loc[subset["profile"] == profile, column], errors="coerce"
|
||||
).dropna().tolist()
|
||||
for j, value in enumerate(values):
|
||||
x.append(index + offset + (j - (len(values) - 1) / 2) * 0.035)
|
||||
y.append(value / scale)
|
||||
ax.scatter(x, y, label=mode, color=color, alpha=0.85)
|
||||
if limit is not None:
|
||||
ax.axhline(limit, color="red", linestyle="--", label="API limit")
|
||||
finish(fig, ax, title + " (each point = one run)", ylabel, name)
|
||||
|
||||
points("api_bytes_per_ready", 1024 ** 2, "API RX+TX per ready attachment",
|
||||
"MiB / ready", "api-traffic")
|
||||
points("api_sampled_peak_window_max", 1024 ** 2,
|
||||
"API sampled memory peak: maximum of two replicas", "MiB",
|
||||
"api-memory-window", limit=512)
|
||||
points("api_peak_lifetime_max", 1024 ** 2,
|
||||
"API lifetime memory peak (includes startup and warmup)", "MiB",
|
||||
"api-memory-lifetime", limit=512)
|
||||
points("p95_seconds", 1, "Successful upload flow p95", "seconds", "latency-p95")
|
||||
points("errors", 1, "Upload errors (missing runs are absent)", "count", "errors")
|
||||
|
||||
fig, ax = plt.subplots(figsize=(10, 5))
|
||||
pairs = summary["decision"].get("pairs", [])
|
||||
for valid, marker, label in [(True, "o", "instrumentation valid"),
|
||||
(False, "x", "instrumentation issues")]:
|
||||
selected = [p for p in pairs if p["instrumentation_valid"] == valid]
|
||||
ax.scatter([profiles.index(p["profile"]) +
|
||||
(p["repetition"] - 2) * 0.06 for p in selected],
|
||||
[p["direct_proxy_ratio"] for p in selected],
|
||||
marker=marker, label=label)
|
||||
ax.axhline(protocol["criteria"]["median_direct_proxy_traffic_ratio_max"],
|
||||
linestyle="--", color="green", label="median threshold")
|
||||
ax.axhline(protocol["criteria"]["each_pair_traffic_ratio_max"],
|
||||
linestyle=":", color="red", label="each pair threshold")
|
||||
finish(fig, ax, "Paired API traffic ratio (all available pairs)",
|
||||
"direct / proxy", "traffic-ratio")
|
||||
provenance = {
|
||||
"created_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"python": platform.python_version(), "pandas": pd.__version__,
|
||||
"matplotlib": matplotlib.__version__,
|
||||
"inputs_sha256": {f.name: sha256(f) for f in inputs + [campaign / "protocol.json"]},
|
||||
"plot_script_sha256": sha256(Path(__file__)),
|
||||
"requirements_sha256": sha256(Path(__file__).parent / "requirements.txt"),
|
||||
"criterion_status": status,
|
||||
"note": "All available runs shown; missing values omitted, never replaced with zero. "
|
||||
"Instrumentation validity and system failures must be read in metrics.json.",
|
||||
}
|
||||
(output / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("campaign", help="prepared campaign under root evidence/upload-comparison")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
print(create_plots(campaign_path(args.campaign)))
|
||||
except (OSError, ValueError, KeyError) as exc:
|
||||
parser.exit(1, "Cannot plot: check campaign path and run summarize first (" +
|
||||
type(exc).__name__ + ").\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,15 @@
|
||||
# Analysis environment (Python 3.12).
|
||||
contourpy==1.4.0
|
||||
cycler==0.12.1
|
||||
fonttools==4.67.0
|
||||
kiwisolver==1.5.1
|
||||
matplotlib==3.9.2
|
||||
numpy==2.5.4
|
||||
packaging==26.3
|
||||
pandas==2.2.3
|
||||
pillow==13.0.0
|
||||
pyparsing==3.3.3
|
||||
python-dateutil==2.9.0.post0
|
||||
pytz==2026.5
|
||||
six==1.17.0
|
||||
tzdata==2026.5
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
"""WSL/Linux entry point. Nothing is measured until the explicit run command."""
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
|
||||
from driver import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
"$PYTHON" -c 'import sys; sys.exit("Use Python 3.12 for the pinned analysis environment") if sys.version_info[:2] != (3, 12) else None'
|
||||
"$PYTHON" -m venv "$ROOT/.venv"
|
||||
"$ROOT/.venv/bin/python" -m pip install --no-compile -r "$ROOT/experiments/requirements.txt"
|
||||
"$ROOT/.venv/bin/python" -m pip check
|
||||
printf '\nReady: %s/.venv/bin/python\n' "$ROOT"
|
||||
@@ -0,0 +1,392 @@
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import statistics
|
||||
|
||||
from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
if not values:
|
||||
return None
|
||||
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
|
||||
|
||||
|
||||
def json_lines(path):
|
||||
if not path.exists():
|
||||
return []
|
||||
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
|
||||
|
||||
|
||||
def case_metrics(directory, protocol):
|
||||
case = load(directory / "case.json")
|
||||
outcome = (
|
||||
load(directory / "outcome.json")
|
||||
if (directory / "outcome.json").exists()
|
||||
else {"status": "interrupted"}
|
||||
)
|
||||
requests, verification = (
|
||||
json_lines(directory / "requests.jsonl"),
|
||||
json_lines(directory / "verification.jsonl"),
|
||||
)
|
||||
expected = protocol["clients"] * protocol["waves"]
|
||||
result = {
|
||||
**case,
|
||||
"status": outcome["status"],
|
||||
"expected_requests": expected,
|
||||
"requests_recorded": len(requests),
|
||||
"ready": sum(r.get("ready", False) for r in requests),
|
||||
"errors": sum(
|
||||
not r.get("ready", False) and r.get("error") != "interrupted"
|
||||
for r in requests
|
||||
),
|
||||
"cancelled": sum(r.get("error") == "interrupted" for r in requests),
|
||||
"verified": sum(v.get("verified", False) for v in verification),
|
||||
"system_failures": [],
|
||||
"instrumentation_issues": [],
|
||||
"containers": [],
|
||||
}
|
||||
system, issues = result["system_failures"], result["instrumentation_issues"]
|
||||
if result["errors"]:
|
||||
system.append("upload_errors")
|
||||
if (
|
||||
len({r["attachment"]["id"] for r in requests if r.get("ready")})
|
||||
!= result["ready"]
|
||||
):
|
||||
system.append("duplicate_attachment_ids")
|
||||
if any(
|
||||
not v.get("verified", False) and v.get("error") != "interrupted"
|
||||
for v in verification
|
||||
):
|
||||
system.append("original_verification_failed")
|
||||
warmups = json_lines(directory / "warmup.jsonl")
|
||||
if any(
|
||||
not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups
|
||||
):
|
||||
system.append("warmup_upload_errors")
|
||||
if len(requests) != expected or result["verified"] != result["ready"]:
|
||||
issues.append("missing_request_or_verification_records")
|
||||
if outcome["status"] != "completed":
|
||||
issues.append("case_" + outcome["status"])
|
||||
if outcome.get("sampler_errors"):
|
||||
issues.append("sampler_error")
|
||||
before = (
|
||||
load(directory / "containers-before.json")
|
||||
if (directory / "containers-before.json").exists()
|
||||
else []
|
||||
)
|
||||
after = (
|
||||
load(directory / "containers-after.json")
|
||||
if (directory / "containers-after.json").exists()
|
||||
else []
|
||||
)
|
||||
for c in after:
|
||||
previous = next((p for p in before if p["id"] == c["id"]), None)
|
||||
if (
|
||||
c["oom_killed"]
|
||||
or c["status"] != "running"
|
||||
or (
|
||||
previous
|
||||
and (
|
||||
c["restarts"] != previous["restarts"]
|
||||
or c["started_at"] != previous["started_at"]
|
||||
)
|
||||
)
|
||||
):
|
||||
system.append("container_oom_restart_or_exit:" + c["service"])
|
||||
if not (directory / "measurement.json").exists():
|
||||
issues.append("missing_measurement")
|
||||
return result
|
||||
m = load(directory / "measurement.json")
|
||||
seconds = (m["finished_ns"] - m["started_ns"]) / 1e9
|
||||
result.update(
|
||||
window_seconds=seconds,
|
||||
generator_cpu_cores=m["generator_cpu_seconds"] / seconds,
|
||||
ready_rps=result["ready"] / seconds,
|
||||
)
|
||||
durations = [
|
||||
(r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready")
|
||||
]
|
||||
result["latency_seconds"] = {
|
||||
"p50": percentile(durations, 0.50),
|
||||
"p95": percentile(durations, 0.95),
|
||||
"p99": percentile(durations, 0.99),
|
||||
}
|
||||
spreads = []
|
||||
for wave in range(protocol["waves"]):
|
||||
starts = [r["started_ns"] for r in requests if r["wave"] == wave]
|
||||
if len(starts) == protocol["clients"]:
|
||||
spreads.append((max(starts) - min(starts)) / 1e9)
|
||||
result["max_start_spread_seconds"] = max(spreads, default=None)
|
||||
criteria = protocol["criteria"]
|
||||
if len(spreads) != protocol["waves"] or any(
|
||||
s > criteria["start_spread_seconds_max"] for s in spreads
|
||||
):
|
||||
issues.append("burst_start_spread")
|
||||
if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]:
|
||||
issues.append("possible_generator_cpu_bottleneck")
|
||||
instances = {r["attachment"]["instance"] for r in requests if r.get("ready")}
|
||||
if len(instances) != protocol["resources"]["api_replicas"]:
|
||||
issues.append("not_all_api_replicas_exercised")
|
||||
warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")}
|
||||
if len(warm_instances) != protocol["resources"]["api_replicas"]:
|
||||
issues.append("not_all_api_replicas_warmed")
|
||||
samples = json_lines(directory / "samples.jsonl")
|
||||
api_traffic, background = 0, 0.0
|
||||
for c in before:
|
||||
cid = c["id"]
|
||||
if cid not in m["before"] or cid not in m["after"]:
|
||||
issues.append("missing_counter_snapshot:" + c["service"])
|
||||
continue
|
||||
start, end = m["before"][cid], m["after"][cid]
|
||||
points = [
|
||||
s
|
||||
for s in samples
|
||||
if s["container"] == cid
|
||||
and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"]
|
||||
]
|
||||
times = (
|
||||
[m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]]
|
||||
)
|
||||
gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])]
|
||||
coverage = len(points) * protocol["sample_interval_seconds"] / seconds
|
||||
rx, tx = (
|
||||
end["rx_bytes"] - start["rx_bytes"],
|
||||
end["tx_bytes"] - start["tx_bytes"],
|
||||
)
|
||||
if rx < 0 or tx < 0:
|
||||
issues.append("network_counter_reset:" + c["service"])
|
||||
idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid]
|
||||
idle_seconds = (
|
||||
idle_end["read_started_ns"] - idle_start["read_started_ns"]
|
||||
) / 1e9
|
||||
idle_rate = (
|
||||
(idle_end["rx_bytes"] + idle_end["tx_bytes"])
|
||||
- (idle_start["rx_bytes"] + idle_start["tx_bytes"])
|
||||
) / idle_seconds
|
||||
cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"]
|
||||
peak = max(
|
||||
[s["memory_current"] for s in points]
|
||||
+ [start["memory.current"], end["memory.current"]]
|
||||
)
|
||||
events = {
|
||||
key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0)
|
||||
for key in ("oom", "oom_kill", "max")
|
||||
}
|
||||
if end["memory.events"].get("oom", 0) or end["memory.events"].get(
|
||||
"oom_kill", 0
|
||||
):
|
||||
system.append("oom_event:" + c["service"])
|
||||
if (
|
||||
coverage < criteria["sampler_coverage_fraction_min"]
|
||||
or max(gaps) > criteria["sampler_gap_seconds_max"]
|
||||
):
|
||||
issues.append("sampler_coverage:" + c["service"])
|
||||
item = {
|
||||
"service": c["service"],
|
||||
"hostname": c["hostname"],
|
||||
"rx_bytes": rx,
|
||||
"tx_bytes": tx,
|
||||
"cpu_seconds": cpu_usec / 1e6,
|
||||
"cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9),
|
||||
"throttled_seconds": (
|
||||
end["cpu.stat"].get("throttled_usec", 0)
|
||||
- start["cpu.stat"].get("throttled_usec", 0)
|
||||
)
|
||||
/ 1e6,
|
||||
"throttled_periods": end["cpu.stat"].get("nr_throttled", 0)
|
||||
- start["cpu.stat"].get("nr_throttled", 0),
|
||||
"baseline_memory": start["memory.current"],
|
||||
"sampled_peak_window": peak,
|
||||
"peak_lifetime_before": start["memory.peak"],
|
||||
"peak_lifetime_after": end["memory.peak"],
|
||||
"memory_limit": end["memory.max"],
|
||||
"memory_events_delta": events,
|
||||
"idle_bytes_per_second": idle_rate,
|
||||
"sample_count": len(points),
|
||||
"sample_coverage": coverage,
|
||||
"sample_gap_max_seconds": max(gaps),
|
||||
}
|
||||
result["containers"].append(item)
|
||||
if c["service"] == "api":
|
||||
api_traffic += rx + tx
|
||||
background += idle_rate * seconds
|
||||
if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]:
|
||||
system.append("api_lifetime_peak_reached_limit")
|
||||
if sum(c["service"] == "api" for c in result["containers"]) != 2:
|
||||
issues.append("missing_api_counters")
|
||||
result["api_rx_tx_bytes"] = api_traffic
|
||||
result["api_bytes_per_ready"] = (
|
||||
api_traffic / result["ready"] if result["ready"] else None
|
||||
)
|
||||
api_metrics = [c for c in result["containers"] if c["service"] == "api"]
|
||||
result["api_sampled_peak_window_max"] = max(
|
||||
(c["sampled_peak_window"] for c in api_metrics), default=None
|
||||
)
|
||||
result["api_peak_lifetime_max"] = max(
|
||||
(c["peak_lifetime_after"] for c in api_metrics), default=None
|
||||
)
|
||||
result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics)
|
||||
result["s3_rx_tx_bytes"] = sum(
|
||||
c["rx_bytes"] + c["tx_bytes"]
|
||||
for c in result["containers"]
|
||||
if c["service"] == "s3"
|
||||
)
|
||||
result["estimated_background_fraction"] = background / (
|
||||
expected * case["size_bytes"]
|
||||
)
|
||||
if (
|
||||
result["estimated_background_fraction"]
|
||||
> criteria["idle_background_payload_fraction_max"]
|
||||
):
|
||||
issues.append("excessive_background_traffic")
|
||||
result["system_failures"] = sorted(set(system))
|
||||
result["instrumentation_issues"] = sorted(set(issues))
|
||||
return result
|
||||
|
||||
|
||||
def decision(protocol, cases, plan):
|
||||
pairs, issues, failures, profiles = [], [], [], []
|
||||
if len(cases) != len(plan):
|
||||
issues.append("not_all_planned_cases_finished")
|
||||
for c in cases:
|
||||
failures.extend(c["system_failures"])
|
||||
issues.extend(c["instrumentation_issues"])
|
||||
for profile in protocol["profiles"]:
|
||||
valid_ratios = []
|
||||
for repetition in range(1, protocol["repetitions"] + 1):
|
||||
pair = [
|
||||
c
|
||||
for c in cases
|
||||
if c["profile"] == profile["name"] and c["repetition"] == repetition
|
||||
]
|
||||
modes = {c["mode"]: c for c in pair}
|
||||
if set(modes) != {"proxy", "direct"} or not all(
|
||||
m.get("api_bytes_per_ready") for m in modes.values()
|
||||
):
|
||||
issues.append("incomplete_pair:" + profile["name"])
|
||||
continue
|
||||
ratio = (
|
||||
modes["direct"]["api_bytes_per_ready"]
|
||||
/ modes["proxy"]["api_bytes_per_ready"]
|
||||
)
|
||||
valid = all(not m["instrumentation_issues"] for m in modes.values())
|
||||
if valid:
|
||||
valid_ratios.append(ratio)
|
||||
pairs.append(
|
||||
{
|
||||
"profile": profile["name"],
|
||||
"repetition": repetition,
|
||||
"direct_proxy_ratio": ratio,
|
||||
"instrumentation_valid": valid,
|
||||
}
|
||||
)
|
||||
profiles.append(
|
||||
{
|
||||
"profile": profile["name"],
|
||||
"valid_pairs": len(valid_ratios),
|
||||
"median_traffic_ratio": statistics.median(valid_ratios)
|
||||
if valid_ratios
|
||||
else None,
|
||||
"min_traffic_ratio": min(valid_ratios, default=None),
|
||||
"max_traffic_ratio": max(valid_ratios, default=None),
|
||||
}
|
||||
)
|
||||
if any(
|
||||
r > protocol["criteria"]["each_pair_traffic_ratio_max"]
|
||||
for r in valid_ratios
|
||||
) or (
|
||||
len(valid_ratios) == protocol["repetitions"]
|
||||
and statistics.median(valid_ratios)
|
||||
> protocol["criteria"]["median_direct_proxy_traffic_ratio_max"]
|
||||
):
|
||||
failures.append("traffic_threshold:" + profile["name"])
|
||||
if protocol["purpose"] == "pilot":
|
||||
status = "pilot_not_for_hypothesis"
|
||||
elif failures:
|
||||
# OOM, errors and incorrect bytes are contrary outcomes, never exclusions.
|
||||
status = "rejected"
|
||||
elif issues:
|
||||
status = "inconclusive"
|
||||
else:
|
||||
status = "confirmed_in_tested_conditions"
|
||||
return {
|
||||
"criterion_status": status,
|
||||
"system_or_threshold_failures": sorted(set(failures)),
|
||||
"instrumentation_issues": sorted(set(issues)),
|
||||
"pairs": pairs,
|
||||
"profiles": profiles,
|
||||
}
|
||||
|
||||
|
||||
def summarize(campaign):
|
||||
manifest = load(campaign / "source-manifest.json")
|
||||
analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"]
|
||||
if any(
|
||||
manifest.get(str(path.relative_to(ROOT))) != digest(path)
|
||||
for path in analysis_sources
|
||||
):
|
||||
raise ExperimentError(
|
||||
"analysis code differs from frozen snapshot; use the archived experiment source"
|
||||
)
|
||||
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
|
||||
cases = []
|
||||
for c in plan:
|
||||
directory = campaign / "cases" / c["directory"]
|
||||
if (directory / "case.json").exists():
|
||||
cases.append(case_metrics(directory, protocol))
|
||||
result = {
|
||||
"protocol_version": protocol["version"],
|
||||
"purpose": protocol["purpose"],
|
||||
"decision": decision(protocol, cases, plan),
|
||||
"cases": cases,
|
||||
}
|
||||
if (campaign / "execution-failure.json").exists():
|
||||
result["decision"]["instrumentation_issues"].append(
|
||||
"campaign_execution_failure"
|
||||
)
|
||||
if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions":
|
||||
result["decision"]["criterion_status"] = "inconclusive"
|
||||
save(campaign / "metrics.json", result)
|
||||
save(
|
||||
campaign / "analysis-provenance.json",
|
||||
{
|
||||
"source_sha256": {
|
||||
str(p.relative_to(ROOT)): digest(p) for p in analysis_sources
|
||||
},
|
||||
"seal_sha256": digest(campaign / "seal.json"),
|
||||
},
|
||||
)
|
||||
with (campaign / "metrics.csv").open("w", newline="") as output:
|
||||
columns = [
|
||||
"profile",
|
||||
"mode",
|
||||
"repetition",
|
||||
"ready",
|
||||
"errors",
|
||||
"verified",
|
||||
"api_bytes_per_ready",
|
||||
"api_sampled_peak_window_max",
|
||||
"api_peak_lifetime_max",
|
||||
"api_cpu_seconds_total",
|
||||
"s3_rx_tx_bytes",
|
||||
"window_seconds",
|
||||
"ready_rps",
|
||||
"generator_cpu_cores",
|
||||
"p50_seconds",
|
||||
"p95_seconds",
|
||||
"p99_seconds",
|
||||
]
|
||||
writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
writer.writerows(
|
||||
{
|
||||
**case,
|
||||
**{
|
||||
name + "_seconds": value
|
||||
for name, value in case.get("latency_seconds", {}).items()
|
||||
},
|
||||
}
|
||||
for case in cases
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,273 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
class FlowError(Exception):
|
||||
def __init__(self, code):
|
||||
self.code = code
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base, protocol, stop):
|
||||
self.base, self.protocol, self.stop = base, protocol, stop
|
||||
|
||||
def request(
|
||||
self,
|
||||
method,
|
||||
url,
|
||||
body=None,
|
||||
token=None,
|
||||
headers=None,
|
||||
stage="api",
|
||||
records=None,
|
||||
deadline=None,
|
||||
):
|
||||
if self.stop.is_set():
|
||||
raise FlowError("interrupted")
|
||||
parsed = urlsplit(url)
|
||||
if parsed.scheme != "http" or parsed.hostname not in ("localhost", "127.0.0.1"):
|
||||
raise FlowError("unexpected_endpoint")
|
||||
expected_port = self.protocol["ports"]["s3" if stage == "s3_post" else "api"]
|
||||
if parsed.port != expected_port:
|
||||
raise FlowError("unexpected_endpoint_port")
|
||||
data = json.dumps(body).encode("utf-8") if isinstance(body, dict) else body
|
||||
h = dict(headers or {})
|
||||
if isinstance(body, dict):
|
||||
h["Content-Type"] = "application/json"
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
remaining = (
|
||||
deadline - time.monotonic()
|
||||
if deadline
|
||||
else self.protocol["request_timeout_seconds"]
|
||||
)
|
||||
if remaining <= 0:
|
||||
raise FlowError("flow_timeout")
|
||||
connection = http.client.HTTPConnection(
|
||||
parsed.hostname,
|
||||
parsed.port,
|
||||
timeout=min(self.protocol["request_timeout_seconds"], remaining),
|
||||
)
|
||||
started = time.monotonic_ns()
|
||||
record = {
|
||||
"stage": stage,
|
||||
"method": method,
|
||||
"started_ns": started,
|
||||
"request_body_bytes": len(data or b""),
|
||||
}
|
||||
if records is not None:
|
||||
records.append(record)
|
||||
try:
|
||||
connection.request(
|
||||
method,
|
||||
parsed.path + (("?" + parsed.query) if parsed.query else ""),
|
||||
data,
|
||||
h,
|
||||
)
|
||||
response = connection.getresponse()
|
||||
payload = response.read(65537)
|
||||
record.update(
|
||||
status=response.status,
|
||||
response_body_bytes=len(payload),
|
||||
instance=response.getheader("X-Instance-Id"),
|
||||
request_id=response.getheader("X-Request-Id"),
|
||||
)
|
||||
if len(payload) > 65536:
|
||||
raise FlowError("unexpected_response_size")
|
||||
try:
|
||||
decoded = json.loads(payload) if payload else None
|
||||
except ValueError:
|
||||
if response.status < 400:
|
||||
raise FlowError("unexpected_response_format") from None
|
||||
decoded = None
|
||||
if response.status >= 400:
|
||||
code = (
|
||||
decoded.get("error", {}).get("code", "http_error")
|
||||
if isinstance(decoded, dict)
|
||||
else "http_" + str(response.status)
|
||||
)
|
||||
record["error"] = code
|
||||
raise FlowError(code)
|
||||
return decoded, response.status, record
|
||||
except FlowError:
|
||||
raise
|
||||
except Exception as error:
|
||||
record["error"] = type(error).__name__
|
||||
raise FlowError(type(error).__name__) from None
|
||||
finally:
|
||||
record["finished_ns"] = time.monotonic_ns()
|
||||
connection.close()
|
||||
|
||||
def api(self, method, path, **kwargs):
|
||||
return self.request(method, self.base + path, **kwargs)
|
||||
|
||||
def bootstrap(self):
|
||||
actors = []
|
||||
for label in ("alice", "bob"):
|
||||
username = "exp_" + label + "_" + secrets.token_hex(6)
|
||||
password = secrets.token_hex(24)
|
||||
user, status, _ = self.api(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": username,
|
||||
"display_name": label,
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
if status != 201:
|
||||
raise FlowError("registration_status")
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
session, status, _ = self.api(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": "Basic " + basic},
|
||||
)
|
||||
if status != 201:
|
||||
raise FlowError("session_status")
|
||||
actors.append({"id": user["id"], "token": session["token"]})
|
||||
chat, status, _ = self.api(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
token=actors[0]["token"],
|
||||
body={
|
||||
"title": "Upload experiment",
|
||||
"member_ids": [a["id"] for a in actors],
|
||||
},
|
||||
)
|
||||
if status != 201:
|
||||
raise FlowError("chat_status")
|
||||
return actors, chat["id"]
|
||||
|
||||
def upload(self, mode, actor, chat, data, expected_digest, records, deadline):
|
||||
path = "/api/v1/chats/" + chat
|
||||
if mode == "proxy":
|
||||
attachment, status, record = self.api(
|
||||
"POST",
|
||||
path + "/attachments?filename=experiment.png",
|
||||
body=data,
|
||||
token=actor["token"],
|
||||
headers={"Content-Type": "image/png"},
|
||||
stage="proxy",
|
||||
records=records,
|
||||
deadline=deadline,
|
||||
)
|
||||
else:
|
||||
# Matches the browser: hash inside the measured logical direct flow.
|
||||
start_hash = time.monotonic_ns()
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
records.append(
|
||||
{
|
||||
"stage": "client_sha256",
|
||||
"started_ns": start_hash,
|
||||
"finished_ns": time.monotonic_ns(),
|
||||
}
|
||||
)
|
||||
instruction, status, _ = self.api(
|
||||
"POST",
|
||||
path + "/attachment-uploads",
|
||||
token=actor["token"],
|
||||
body={
|
||||
"filename": "experiment.png",
|
||||
"content_type": "image/png",
|
||||
"size_bytes": len(data),
|
||||
"sha256": checksum,
|
||||
},
|
||||
stage="init",
|
||||
records=records,
|
||||
deadline=deadline,
|
||||
)
|
||||
if status != 201:
|
||||
raise FlowError("init_status")
|
||||
fields = instruction["upload"]["fields"]
|
||||
boundary = "experiment-" + secrets.token_hex(12)
|
||||
pieces = [
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode()
|
||||
for k, v in fields.items()
|
||||
]
|
||||
pieces += [
|
||||
f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="experiment.png"\r\nContent-Type: image/png\r\n\r\n'.encode(),
|
||||
data,
|
||||
f"\r\n--{boundary}--\r\n".encode(),
|
||||
]
|
||||
multipart = b"".join(pieces)
|
||||
# This is the only S3 request; no Bearer token or application headers.
|
||||
_, post_status, _ = self.request(
|
||||
"POST",
|
||||
instruction["upload"]["url"],
|
||||
body=multipart,
|
||||
headers={"Content-Type": "multipart/form-data; boundary=" + boundary},
|
||||
stage="s3_post",
|
||||
records=records,
|
||||
deadline=deadline,
|
||||
)
|
||||
if post_status != 204:
|
||||
raise FlowError("s3_post_status")
|
||||
attachment, status, record = self.api(
|
||||
"POST",
|
||||
path + "/attachment-uploads/" + instruction["upload_id"] + "/complete",
|
||||
token=actor["token"],
|
||||
stage="complete",
|
||||
records=records,
|
||||
deadline=deadline,
|
||||
)
|
||||
if (
|
||||
status != 201
|
||||
or attachment.get("status") != "ready"
|
||||
or attachment.get("size_bytes") != len(data)
|
||||
or attachment.get("sha256") != expected_digest
|
||||
):
|
||||
raise FlowError("incorrect_ready_metadata")
|
||||
return {
|
||||
"id": attachment["id"],
|
||||
"sha256": attachment["sha256"],
|
||||
"size_bytes": attachment["size_bytes"],
|
||||
"instance": record["instance"],
|
||||
}
|
||||
|
||||
def verify(self, actor, attachment, size, expected_digest):
|
||||
if self.stop.is_set():
|
||||
raise FlowError("interrupted")
|
||||
parsed = urlsplit(self.base)
|
||||
connection = http.client.HTTPConnection(
|
||||
parsed.hostname, parsed.port, timeout=35
|
||||
)
|
||||
checksum, received = hashlib.sha256(), 0
|
||||
try:
|
||||
connection.request(
|
||||
"GET",
|
||||
"/api/v1/attachments/" + attachment["id"] + "/content",
|
||||
headers={"Authorization": "Bearer " + actor["token"]},
|
||||
)
|
||||
response = connection.getresponse()
|
||||
if (
|
||||
response.status != 200
|
||||
or response.getheader("Content-Length") != str(size)
|
||||
or response.getheader("Content-Type") != "image/png"
|
||||
):
|
||||
raise FlowError("download_headers")
|
||||
while True:
|
||||
if self.stop.is_set():
|
||||
raise FlowError("interrupted")
|
||||
block = response.read(32768)
|
||||
if not block:
|
||||
break
|
||||
received += len(block)
|
||||
if received > size:
|
||||
raise FlowError("download_size")
|
||||
checksum.update(block)
|
||||
if received != size or checksum.hexdigest() != expected_digest:
|
||||
raise FlowError("download_checksum")
|
||||
return {
|
||||
"id": attachment["id"],
|
||||
"verified": True,
|
||||
"size_bytes": received,
|
||||
"sha256": checksum.hexdigest(),
|
||||
}
|
||||
finally:
|
||||
connection.close()
|
||||
@@ -0,0 +1,112 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
EXPERIMENT = Path(__file__).resolve().parents[1]
|
||||
ROOT = EXPERIMENT.parent
|
||||
EVIDENCE = ROOT / "evidence" / "upload-comparison"
|
||||
|
||||
|
||||
class ExperimentError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def digest(path):
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def save(path, value):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(
|
||||
json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def load(path):
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def command(args, log=None, timeout=300, input_text=None):
|
||||
# Never format the exception with subprocess output: Docker can include
|
||||
# environment details. Explicit build/start logs use commands without secrets.
|
||||
result = subprocess.run(
|
||||
args,
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
input=input_text,
|
||||
stdout=log or subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode:
|
||||
raise ExperimentError(
|
||||
"command failed: " + args[0] + " (exit " + str(result.returncode) + ")"
|
||||
)
|
||||
return result.stdout.strip() if log is None else ""
|
||||
|
||||
|
||||
def source_files():
|
||||
fixed = [
|
||||
ROOT / "src/Dockerfile",
|
||||
ROOT / "src/.dockerignore",
|
||||
ROOT / "src/go.mod",
|
||||
ROOT / "src/go.sum",
|
||||
ROOT / "infra/minio.Dockerfile",
|
||||
ROOT / "infra/haproxy.cfg",
|
||||
]
|
||||
app = sorted(
|
||||
p
|
||||
for p in (ROOT / "src/backend").rglob("*")
|
||||
if p.is_file() and p.suffix in (".go", ".sql")
|
||||
)
|
||||
harness = sorted(
|
||||
p
|
||||
for p in EXPERIMENT.rglob("*")
|
||||
if p.is_file()
|
||||
and "evidence" not in p.relative_to(EXPERIMENT).parts
|
||||
and "__pycache__" not in p.parts
|
||||
and p.suffix != ".pyc"
|
||||
)
|
||||
return sorted(set(fixed + app + harness))
|
||||
|
||||
|
||||
def source_manifest():
|
||||
return {str(p.relative_to(ROOT)): digest(p) for p in source_files()}
|
||||
|
||||
|
||||
def check_sources(campaign):
|
||||
expected = load(campaign / "source-manifest.json")
|
||||
if source_manifest() != expected:
|
||||
raise ExperimentError(
|
||||
"source/config/docs changed since prepare; create a new campaign"
|
||||
)
|
||||
|
||||
|
||||
def check_seal(campaign):
|
||||
seal = load(campaign / "seal.json")
|
||||
for name, expected in seal.items():
|
||||
if digest(campaign / name) != expected:
|
||||
raise ExperimentError("frozen artifact changed: " + name)
|
||||
for entry in load(campaign / "fixtures.json"):
|
||||
if digest(campaign / "fixtures" / entry["filename"]) != entry["sha256"]:
|
||||
raise ExperimentError("fixture checksum changed")
|
||||
|
||||
|
||||
def campaign_path(value):
|
||||
path = Path(value).expanduser().resolve()
|
||||
if path.parent != EVIDENCE.resolve() or not (path / "protocol.json").is_file():
|
||||
raise ExperimentError(
|
||||
"campaign must be a prepared directory directly inside evidence/upload-comparison"
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def private_directory():
|
||||
path = Path(tempfile.mkdtemp(prefix="highload-upload-exp-", dir="/tmp"))
|
||||
path.chmod(0o700)
|
||||
return path
|
||||
@@ -0,0 +1,833 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import resource
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
from analysis import summarize
|
||||
from client import Client, FlowError
|
||||
from common import (
|
||||
EVIDENCE,
|
||||
EXPERIMENT,
|
||||
ROOT,
|
||||
ExperimentError,
|
||||
campaign_path,
|
||||
check_seal,
|
||||
check_sources,
|
||||
command,
|
||||
digest,
|
||||
load,
|
||||
private_directory,
|
||||
save,
|
||||
source_manifest,
|
||||
)
|
||||
from fixtures import make_png
|
||||
from telemetry import Sampler, inspect_containers, snapshot
|
||||
|
||||
|
||||
def make_plan(protocol):
|
||||
blocks = []
|
||||
for index, profile in enumerate(protocol["profiles"]):
|
||||
for repetition in range(1, protocol["repetitions"] + 1):
|
||||
modes = (
|
||||
["proxy", "direct"] if (index + repetition) % 2 else ["direct", "proxy"]
|
||||
)
|
||||
blocks.append((profile, repetition, modes))
|
||||
blocks.sort(
|
||||
key=lambda b: hashlib.sha256(
|
||||
f"{protocol['seed']}:{b[0]['name']}:{b[1]}".encode()
|
||||
).digest()
|
||||
)
|
||||
plan = []
|
||||
for profile, repetition, modes in blocks:
|
||||
for mode in modes:
|
||||
number = len(plan) + 1
|
||||
plan.append(
|
||||
{
|
||||
"number": number,
|
||||
"profile": profile["name"],
|
||||
"mode": mode,
|
||||
"repetition": repetition,
|
||||
"size_bytes": profile["size_bytes"],
|
||||
"directory": f"{number:02d}-{profile['name']}-{mode}-r{repetition}",
|
||||
}
|
||||
)
|
||||
return plan
|
||||
|
||||
|
||||
def host_info():
|
||||
info = json.loads(command(["docker", "info", "--format", "{{json .}}"], timeout=30))
|
||||
if str(info.get("CgroupVersion")) != "2":
|
||||
raise ExperimentError("Docker cgroup v2 is required")
|
||||
safe = {
|
||||
key: info.get(key)
|
||||
for key in (
|
||||
"ServerVersion",
|
||||
"CgroupVersion",
|
||||
"NCPU",
|
||||
"MemTotal",
|
||||
"KernelVersion",
|
||||
"OperatingSystem",
|
||||
"Driver",
|
||||
)
|
||||
}
|
||||
return {
|
||||
"python": sys.version,
|
||||
"zlib": zlib.ZLIB_VERSION,
|
||||
"platform": platform.platform(),
|
||||
"docker": safe,
|
||||
"compose": command(["docker", "compose", "version", "--short"]),
|
||||
"client_cpu_affinity_count": len(os.sched_getaffinity(0))
|
||||
if hasattr(os, "sched_getaffinity")
|
||||
else os.cpu_count(),
|
||||
"disk_free_bytes": shutil.disk_usage(EXPERIMENT).free,
|
||||
}
|
||||
|
||||
|
||||
def check_ports(protocol):
|
||||
for port in protocol["ports"].values():
|
||||
with socket.socket() as test:
|
||||
try:
|
||||
test.bind(("127.0.0.1", port))
|
||||
except OSError:
|
||||
raise ExperimentError(
|
||||
"experiment port is occupied: " + str(port)
|
||||
) from None
|
||||
|
||||
|
||||
def preflight():
|
||||
if platform.system() != "Linux":
|
||||
raise ExperimentError("run inside WSL Ubuntu/Linux, not Windows Python")
|
||||
protocol = load(EXPERIMENT / "config/protocol.json")
|
||||
validate_compose(protocol)
|
||||
info = host_info()
|
||||
check_ports(protocol)
|
||||
if not command(["docker", "compose", "version", "--short"]):
|
||||
raise ExperimentError("Docker Compose plugin is required")
|
||||
return info
|
||||
|
||||
|
||||
def validate_compose(protocol):
|
||||
if len(set(protocol["ports"].values())) != 3:
|
||||
raise ExperimentError("API/S3/stats ports must differ")
|
||||
values = {
|
||||
"REPO_ROOT": str(ROOT),
|
||||
"SECRET_DIR": "/tmp/experiment-config-check",
|
||||
**{
|
||||
role + "_IMAGE": "sha256:" + "0" * 64
|
||||
for role in ("APP", "S3", "DB", "REDIS", "HAPROXY")
|
||||
},
|
||||
**{
|
||||
role.upper() + "_PORT": str(port)
|
||||
for role, port in protocol["ports"].items()
|
||||
},
|
||||
}
|
||||
response = subprocess.run(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
"/dev/null",
|
||||
"--project-name",
|
||||
"hl-upload-exp-config-check",
|
||||
"-f",
|
||||
str(EXPERIMENT / "config/compose.yml"),
|
||||
"config",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
check=False,
|
||||
cwd=ROOT,
|
||||
env={**os.environ, **values},
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=30,
|
||||
)
|
||||
if response.returncode:
|
||||
raise ExperimentError("experiment Compose config is invalid")
|
||||
services = json.loads(response.stdout)["services"]
|
||||
resources = protocol["resources"]
|
||||
for role in ("api", "db", "s3", "redis", "haproxy"):
|
||||
if (
|
||||
float(services[role]["cpus"]) != resources[role + "_cpus"]
|
||||
or int(services[role]["mem_limit"]) != resources[role + "_memory_bytes"]
|
||||
):
|
||||
raise ExperimentError("protocol/Compose resource mismatch: " + role)
|
||||
if int(services[role]["memswap_limit"]) != int(services[role]["mem_limit"]):
|
||||
raise ExperimentError("experiment swap must be disabled: " + role)
|
||||
api = services["api"]
|
||||
expected_env = {
|
||||
"GOMAXPROCS": str(resources["api_gomaxprocs"]),
|
||||
"GOMEMLIMIT": resources["api_go_memory_limit"],
|
||||
"DATABASE_MAX_CONNS": str(resources["database_max_connections_per_replica"]),
|
||||
}
|
||||
if api["deploy"]["replicas"] != resources["api_replicas"] or any(
|
||||
api["environment"].get(k) != v for k, v in expected_env.items()
|
||||
):
|
||||
raise ExperimentError("protocol/Compose API environment mismatch")
|
||||
if (
|
||||
api["tmpfs"] != ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"]
|
||||
or resources["api_tmpfs_bytes"] != 16777216
|
||||
):
|
||||
raise ExperimentError("unexpected tmpfs configuration")
|
||||
|
||||
|
||||
def prepare(pilot=False, reuse=None):
|
||||
info = preflight()
|
||||
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
||||
stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
campaign = EVIDENCE / (
|
||||
stamp + "-" + ("pilot" if pilot else "formal") + "-" + secrets.token_hex(3)
|
||||
)
|
||||
campaign.mkdir()
|
||||
protocol = load(EXPERIMENT / "config/protocol.json")
|
||||
if pilot:
|
||||
protocol.update(
|
||||
purpose="pilot", repetitions=1, clients=2, waves=1, warmup_clients=2
|
||||
)
|
||||
protocol["profiles"] = [p for p in protocol["profiles"] if p["width"] == 1024]
|
||||
# Frozen before builds or any application requests.
|
||||
save(campaign / "protocol.json", protocol)
|
||||
save(campaign / "plan.json", make_plan(protocol))
|
||||
sources = source_manifest()
|
||||
save(campaign / "source-manifest.json", sources)
|
||||
shutil.copyfile(EXPERIMENT / "config/compose.yml", campaign / "compose.yml")
|
||||
with tarfile.open(campaign / "source.tar.gz", "w:gz") as archive:
|
||||
for name in sources:
|
||||
archive.add(ROOT / name, arcname=name, recursive=False)
|
||||
save(campaign / "host-prepare.json", info)
|
||||
save(
|
||||
campaign / "git.json",
|
||||
{
|
||||
"head": command(["git", "rev-parse", "HEAD"]),
|
||||
"branch": command(["git", "branch", "--show-current"]),
|
||||
"status": command(["git", "status", "--short", "--untracked-files=no"]),
|
||||
},
|
||||
)
|
||||
fixtures = []
|
||||
for p in protocol["profiles"]:
|
||||
filename = p["name"] + ".png"
|
||||
data = make_png(p["width"], p["height"], p["size_bytes"], protocol["seed"])
|
||||
(campaign / "fixtures").mkdir(exist_ok=True)
|
||||
(campaign / "fixtures" / filename).write_bytes(data)
|
||||
fixtures.append(
|
||||
{**p, "filename": filename, "sha256": hashlib.sha256(data).hexdigest()}
|
||||
)
|
||||
save(campaign / "fixtures.json", fixtures)
|
||||
try:
|
||||
if reuse:
|
||||
check_seal(reuse)
|
||||
if load(reuse / "source-manifest.json") != sources:
|
||||
raise ExperimentError(
|
||||
"reuse-images source manifest differs; prepare without reuse"
|
||||
)
|
||||
images = load(reuse / "images.json")
|
||||
for image in images.values():
|
||||
command(["docker", "image", "inspect", image["id"]])
|
||||
else:
|
||||
refs = dict(protocol["images"])
|
||||
app_hash = hashlib.sha256(
|
||||
json.dumps(
|
||||
{k: v for k, v in sources.items() if k.startswith("src/")},
|
||||
sort_keys=True,
|
||||
).encode()
|
||||
).hexdigest()[:16]
|
||||
refs["app"] = "highload-upload-experiment-app:" + app_hash
|
||||
refs["s3"] = (
|
||||
"highload-upload-experiment-s3:"
|
||||
+ sources["infra/minio.Dockerfile"][:16]
|
||||
)
|
||||
with (campaign / "prepare.log").open("w") as log:
|
||||
for role in ("db", "redis", "haproxy"):
|
||||
try:
|
||||
command(["docker", "image", "inspect", refs[role]])
|
||||
except ExperimentError:
|
||||
command(["docker", "pull", refs[role]], log, timeout=600)
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"build",
|
||||
"--tag",
|
||||
refs["app"],
|
||||
"--file",
|
||||
str(ROOT / "src/Dockerfile"),
|
||||
str(ROOT / "src"),
|
||||
],
|
||||
log,
|
||||
timeout=1800,
|
||||
)
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"build",
|
||||
"--tag",
|
||||
refs["s3"],
|
||||
"--file",
|
||||
str(ROOT / "infra/minio.Dockerfile"),
|
||||
str(ROOT / "infra"),
|
||||
],
|
||||
log,
|
||||
timeout=1800,
|
||||
)
|
||||
images = {}
|
||||
for role, ref in refs.items():
|
||||
image = json.loads(command(["docker", "image", "inspect", ref]))[0]
|
||||
images[role] = {
|
||||
"id": image["Id"],
|
||||
"reference": ref,
|
||||
"repo_digests": image.get("RepoDigests", []),
|
||||
"os": image["Os"],
|
||||
"architecture": image["Architecture"],
|
||||
"layers": image["RootFS"]["Layers"],
|
||||
}
|
||||
save(campaign / "images.json", images)
|
||||
check_sources(campaign)
|
||||
names = [
|
||||
"protocol.json",
|
||||
"plan.json",
|
||||
"source-manifest.json",
|
||||
"source.tar.gz",
|
||||
"fixtures.json",
|
||||
"images.json",
|
||||
"compose.yml",
|
||||
]
|
||||
save(campaign / "seal.json", {name: digest(campaign / name) for name in names})
|
||||
save(
|
||||
campaign / "prepared.json",
|
||||
{
|
||||
"status": "prepared",
|
||||
"purpose": protocol["purpose"],
|
||||
"cases": len(load(campaign / "plan.json")),
|
||||
},
|
||||
)
|
||||
except Exception as error:
|
||||
save(
|
||||
campaign / "prepare-failure.json",
|
||||
{"error_type": type(error).__name__, "status": "not_prepared"},
|
||||
)
|
||||
raise
|
||||
print(
|
||||
"Prepared; no uploads measured: " + str(campaign.relative_to(ROOT)), flush=True
|
||||
)
|
||||
print("Seal: " + digest(campaign / "seal.json"), flush=True)
|
||||
return campaign
|
||||
|
||||
|
||||
def runtime_files(campaign):
|
||||
directory = private_directory()
|
||||
credentials = {
|
||||
"db_password": secrets.token_hex(24),
|
||||
"root_access": secrets.token_hex(12),
|
||||
"root_secret": secrets.token_hex(32),
|
||||
"api_access": secrets.token_hex(12),
|
||||
"api_secret": secrets.token_hex(32),
|
||||
}
|
||||
credentials["database_url"] = (
|
||||
"postgres://messenger_exp:"
|
||||
+ credentials["db_password"]
|
||||
+ "@db:5432/messenger_exp?sslmode=disable"
|
||||
)
|
||||
credentials["cache_url"] = "redis://redis:6379/0"
|
||||
for name, value in credentials.items():
|
||||
path = directory / name
|
||||
path.write_text(value + "\n")
|
||||
path.chmod(0o644) # private parent is 0700; bind individual files for app UID.
|
||||
images, protocol = load(campaign / "images.json"), load(campaign / "protocol.json")
|
||||
values = {
|
||||
"REPO_ROOT": str(ROOT),
|
||||
"SECRET_DIR": str(directory),
|
||||
**{role.upper() + "_IMAGE": image["id"] for role, image in images.items()},
|
||||
**{
|
||||
role.upper() + "_PORT": str(port)
|
||||
for role, port in protocol["ports"].items()
|
||||
},
|
||||
}
|
||||
env = directory / "runtime.env"
|
||||
env.write_text("".join(key + "=" + value + "\n" for key, value in values.items()))
|
||||
env.chmod(0o600)
|
||||
save(campaign / "runtime.json", {"private_directory": str(directory)})
|
||||
return directory
|
||||
|
||||
|
||||
def compose(campaign, runtime, project):
|
||||
if not project.startswith("hl-upload-exp-") or len(project) > 63:
|
||||
raise ExperimentError("unsafe experiment project name")
|
||||
return [
|
||||
"docker",
|
||||
"compose",
|
||||
"--project-name",
|
||||
project,
|
||||
"--env-file",
|
||||
str(runtime / "runtime.env"),
|
||||
"-f",
|
||||
str(campaign / "compose.yml"),
|
||||
]
|
||||
|
||||
|
||||
def safe_down(campaign, runtime, project, log, keep_volumes):
|
||||
# Every deletion is scoped by a unique project label, never a global prune.
|
||||
allowed = {
|
||||
project_name(campaign, case["number"]) for case in load(campaign / "plan.json")
|
||||
}
|
||||
if project not in allowed:
|
||||
raise ExperimentError("refusing cleanup of another campaign's project")
|
||||
for kind in ("container", "volume", "network"):
|
||||
args = [
|
||||
"docker",
|
||||
kind,
|
||||
"ls",
|
||||
"--filter",
|
||||
"label=com.docker.compose.project=" + project,
|
||||
"-q",
|
||||
]
|
||||
if kind == "container":
|
||||
args.insert(3, "-a")
|
||||
ids = command(args).splitlines()
|
||||
if ids:
|
||||
items = json.loads(command(["docker", kind, "inspect", *ids]))
|
||||
for item in items:
|
||||
labels = (
|
||||
item.get("Labels")
|
||||
if kind != "container"
|
||||
else item["Config"].get("Labels")
|
||||
)
|
||||
if (labels or {}).get("com.docker.compose.project") != project:
|
||||
raise ExperimentError(
|
||||
"refusing cleanup outside the selected project"
|
||||
)
|
||||
args = compose(campaign, runtime, project) + ["down", "--timeout", "15"]
|
||||
if not keep_volumes:
|
||||
args.append("--volumes")
|
||||
command(args, log, timeout=120)
|
||||
|
||||
|
||||
def project_name(campaign, number):
|
||||
return (
|
||||
"hl-upload-exp-"
|
||||
+ campaign.name.lower().replace("t", "").replace("z", "")
|
||||
+ "-"
|
||||
+ str(number)
|
||||
)
|
||||
|
||||
|
||||
def host_activity():
|
||||
result = {"loadavg": list(os.getloadavg()), "monotonic_ns": time.monotonic_ns()}
|
||||
for row in Path("/proc/meminfo").read_text().splitlines():
|
||||
if row.startswith(("MemAvailable:", "SwapFree:", "SwapTotal:")):
|
||||
key, value = row.split(":", 1)
|
||||
result[key + "_bytes"] = int(value.split()[0]) * 1024
|
||||
return result
|
||||
|
||||
|
||||
def batch(client, mode, actor, chat, data, digest_value, count, wave, output, stop):
|
||||
barrier = threading.Barrier(count + 1)
|
||||
|
||||
def worker(index):
|
||||
records = []
|
||||
barrier.wait(timeout=15)
|
||||
started = time.monotonic_ns()
|
||||
result = {
|
||||
"wave": wave,
|
||||
"client": index,
|
||||
"started_ns": started,
|
||||
"ready": False,
|
||||
"stages": records,
|
||||
}
|
||||
try:
|
||||
attachment = client.upload(
|
||||
mode,
|
||||
actor,
|
||||
chat,
|
||||
data,
|
||||
digest_value,
|
||||
records,
|
||||
time.monotonic() + client.protocol["flow_timeout_seconds"],
|
||||
)
|
||||
result.update(ready=True, attachment=attachment)
|
||||
except FlowError as error:
|
||||
result["error"] = error.code
|
||||
except Exception as error:
|
||||
result["error"] = type(error).__name__
|
||||
result["finished_ns"] = time.monotonic_ns()
|
||||
return result
|
||||
|
||||
results = []
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
|
||||
futures = [pool.submit(worker, index) for index in range(count)]
|
||||
barrier.wait(timeout=15)
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
result = future.result()
|
||||
output.write(json.dumps(result) + "\n")
|
||||
output.flush()
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def run_case(campaign, case, runtime, project, stop):
|
||||
directory = campaign / "cases" / case["directory"]
|
||||
directory.mkdir(parents=True)
|
||||
save(directory / "case.json", {**case, "project": project})
|
||||
protocol, images = load(campaign / "protocol.json"), load(campaign / "images.json")
|
||||
cmd = compose(campaign, runtime, project)
|
||||
sampler, containers, outcome = (
|
||||
None,
|
||||
[],
|
||||
{"status": "setup_failed", "stage": "startup"},
|
||||
)
|
||||
try:
|
||||
save(directory / "host-before.json", host_activity())
|
||||
with (directory / "startup.log").open("w") as log:
|
||||
command(
|
||||
cmd + ["up", "-d", "--no-build", "--wait", "--wait-timeout", "120"],
|
||||
log,
|
||||
timeout=180,
|
||||
)
|
||||
ids = command(cmd + ["ps", "-q", "api", "s3"]).splitlines()
|
||||
containers = inspect_containers(ids, project)
|
||||
apis = [c for c in containers if c["service"] == "api"]
|
||||
if len(apis) != 2 or len(containers) != 3:
|
||||
raise ExperimentError("expected two API containers and one S3 container")
|
||||
for c in containers:
|
||||
expected = images["app" if c["service"] == "api" else "s3"]["id"]
|
||||
if c["image"] != expected:
|
||||
raise ExperimentError("unexpected image ID")
|
||||
for c in apis:
|
||||
if (
|
||||
c["memory_limit"] != protocol["resources"]["api_memory_bytes"]
|
||||
or c["memory_and_swap_limit"] != c["memory_limit"]
|
||||
or c["nano_cpus"] != 500000000
|
||||
or "size=16m" not in c["tmpfs"].get("/tmp/uploads", "")
|
||||
):
|
||||
raise ExperimentError("API resource configuration changed")
|
||||
outcome["stage"] = "bootstrap_warmup"
|
||||
client = Client(
|
||||
"http://127.0.0.1:" + str(protocol["ports"]["api"]), protocol, stop
|
||||
)
|
||||
actors, chat = client.bootstrap()
|
||||
fixture = next(
|
||||
p for p in load(campaign / "fixtures.json") if p["name"] == case["profile"]
|
||||
)
|
||||
data = (campaign / "fixtures" / fixture["filename"]).read_bytes()
|
||||
with (directory / "warmup.jsonl").open("w") as output:
|
||||
warmup = batch(
|
||||
client,
|
||||
case["mode"],
|
||||
actors[0],
|
||||
chat,
|
||||
data,
|
||||
fixture["sha256"],
|
||||
protocol["warmup_clients"],
|
||||
-1,
|
||||
output,
|
||||
stop,
|
||||
)
|
||||
if not all(r["ready"] for r in warmup):
|
||||
raise ExperimentError("warmup upload failed")
|
||||
time.sleep(protocol["settle_seconds"])
|
||||
sampler = Sampler(
|
||||
containers, directory / "samples.jsonl", protocol["sample_interval_seconds"]
|
||||
)
|
||||
sampler.wait_ready()
|
||||
idle_before = snapshot(containers)
|
||||
time.sleep(protocol["idle_seconds"])
|
||||
idle_after = snapshot(containers)
|
||||
save(directory / "containers-before.json", inspect_containers(ids, project))
|
||||
outcome["stage"] = "measured_uploads"
|
||||
started = time.monotonic_ns()
|
||||
cpu_before = resource.getrusage(resource.RUSAGE_SELF)
|
||||
before = snapshot(containers)
|
||||
requests = []
|
||||
with (directory / "requests.jsonl").open("w") as output:
|
||||
for wave in range(protocol["waves"]):
|
||||
if stop.is_set():
|
||||
break
|
||||
requests.extend(
|
||||
batch(
|
||||
client,
|
||||
case["mode"],
|
||||
actors[0],
|
||||
chat,
|
||||
data,
|
||||
fixture["sha256"],
|
||||
protocol["clients"],
|
||||
wave,
|
||||
output,
|
||||
stop,
|
||||
)
|
||||
)
|
||||
if wave + 1 < protocol["waves"]:
|
||||
stop.wait(protocol["wave_pause_seconds"])
|
||||
after = snapshot(containers)
|
||||
cpu_after = resource.getrusage(resource.RUSAGE_SELF)
|
||||
finished = time.monotonic_ns()
|
||||
save(
|
||||
directory / "measurement.json",
|
||||
{
|
||||
"started_ns": started,
|
||||
"finished_ns": finished,
|
||||
"generator_cpu_seconds": cpu_after.ru_utime
|
||||
+ cpu_after.ru_stime
|
||||
- cpu_before.ru_utime
|
||||
- cpu_before.ru_stime,
|
||||
"generator_maxrss_kib": cpu_after.ru_maxrss,
|
||||
"idle_before": idle_before,
|
||||
"idle_after": idle_after,
|
||||
"before": before,
|
||||
"after": after,
|
||||
},
|
||||
)
|
||||
sampler.close()
|
||||
outcome["sampler_errors"] = sampler.errors
|
||||
sampler = None
|
||||
# Downloads must not contaminate measured API network counters.
|
||||
outcome["stage"] = "verify_originals"
|
||||
with (directory / "verification.jsonl").open("w") as output:
|
||||
for request in requests:
|
||||
if stop.is_set():
|
||||
break
|
||||
if not request["ready"]:
|
||||
continue
|
||||
try:
|
||||
verified = client.verify(
|
||||
actors[1], request["attachment"], len(data), fixture["sha256"]
|
||||
)
|
||||
except Exception as error:
|
||||
verified = {
|
||||
"id": request["attachment"]["id"],
|
||||
"verified": False,
|
||||
"error": error.code
|
||||
if isinstance(error, FlowError)
|
||||
else type(error).__name__,
|
||||
}
|
||||
output.write(json.dumps(verified) + "\n")
|
||||
output.flush()
|
||||
outcome.update(
|
||||
status="interrupted" if stop.is_set() else "completed", stage="done"
|
||||
)
|
||||
except Exception as error:
|
||||
outcome["error_type"] = type(error).__name__
|
||||
if outcome["stage"] != "startup":
|
||||
outcome["status"] = "instrumentation_or_setup_failure"
|
||||
finally:
|
||||
if sampler:
|
||||
sampler.close()
|
||||
outcome["sampler_errors"] = sampler.errors
|
||||
if containers:
|
||||
try:
|
||||
save(
|
||||
directory / "containers-after.json",
|
||||
inspect_containers([c["id"] for c in containers], project),
|
||||
)
|
||||
except Exception as error:
|
||||
outcome["inspect_error"] = type(error).__name__
|
||||
save(directory / "host-after.json", host_activity())
|
||||
save(directory / "outcome.json", outcome)
|
||||
return outcome
|
||||
|
||||
|
||||
def remove_runtime(runtime):
|
||||
resolved = runtime.resolve()
|
||||
if resolved.parent != Path("/tmp") or not resolved.name.startswith(
|
||||
"highload-upload-exp-"
|
||||
):
|
||||
raise ExperimentError("refusing to remove unexpected private directory")
|
||||
shutil.rmtree(resolved)
|
||||
|
||||
|
||||
def run_campaign(campaign, resume=False, keep_volumes=False):
|
||||
check_seal(campaign)
|
||||
check_sources(campaign)
|
||||
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
|
||||
check_ports(protocol)
|
||||
save(campaign / "host-run.json", host_info())
|
||||
if (campaign / "cases").exists() and not resume:
|
||||
raise ExperimentError(
|
||||
"campaign already started; use --resume (does not overwrite attempts)"
|
||||
)
|
||||
if (campaign / "run-options.json").exists():
|
||||
if load(campaign / "run-options.json")["keep_volumes"] != keep_volumes:
|
||||
raise ExperimentError(
|
||||
"resume must keep the original --keep-volumes setting; cleanup separately"
|
||||
)
|
||||
else:
|
||||
save(campaign / "run-options.json", {"keep_volumes": keep_volumes})
|
||||
if (campaign / "runtime.json").exists():
|
||||
runtime = Path(load(campaign / "runtime.json")["private_directory"])
|
||||
if not runtime.exists():
|
||||
runtime = runtime_files(campaign)
|
||||
else:
|
||||
runtime = runtime_files(campaign)
|
||||
stop = threading.Event()
|
||||
previous = {}
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
previous[sig] = signal.signal(sig, lambda *_: stop.set())
|
||||
outcomes, cleanup_failed = [], False
|
||||
invocation = (
|
||||
datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
|
||||
+ "-"
|
||||
+ secrets.token_hex(3)
|
||||
)
|
||||
try:
|
||||
for case in plan:
|
||||
if stop.is_set():
|
||||
break
|
||||
directory = campaign / "cases" / case["directory"]
|
||||
if directory.exists():
|
||||
print("Preserved existing attempt: " + case["directory"], flush=True)
|
||||
continue
|
||||
check_sources(campaign)
|
||||
check_seal(campaign)
|
||||
project = project_name(campaign, case["number"])
|
||||
# Refuse reusing a project even after an earlier hard interruption.
|
||||
if command(
|
||||
[
|
||||
"docker",
|
||||
"volume",
|
||||
"ls",
|
||||
"-q",
|
||||
"--filter",
|
||||
"label=com.docker.compose.project=" + project,
|
||||
]
|
||||
):
|
||||
raise ExperimentError(
|
||||
"existing experiment volumes found; cleanup or prepare a new campaign"
|
||||
)
|
||||
print("Running " + case["directory"], flush=True)
|
||||
try:
|
||||
outcome = run_case(campaign, case, runtime, project, stop)
|
||||
outcomes.append({"case": case["directory"], **outcome})
|
||||
finally:
|
||||
try:
|
||||
with (directory / "shutdown.log").open("w") as log:
|
||||
safe_down(campaign, runtime, project, log, keep_volumes)
|
||||
except Exception as error:
|
||||
cleanup_failed = True
|
||||
save(
|
||||
directory / "cleanup-failure.json",
|
||||
{"error_type": type(error).__name__, "project": project},
|
||||
)
|
||||
raise ExperimentError(
|
||||
"cleanup failed; inspect evidence and use cleanup command"
|
||||
) from None
|
||||
execution = {"interrupted": stop.is_set(), "last_invocation": outcomes}
|
||||
save(campaign / "invocations" / (invocation + ".json"), execution)
|
||||
save(campaign / "execution.json", execution)
|
||||
check_sources(campaign)
|
||||
print(
|
||||
"Uploads finished; raw evidence retained. Run summarize; no human conclusion has been written.",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as error:
|
||||
save(
|
||||
campaign / "execution-failure.json",
|
||||
{
|
||||
"error_type": type(error).__name__,
|
||||
"reason": str(error)
|
||||
if isinstance(error, ExperimentError)
|
||||
else "execution interrupted by instrumentation/environment failure",
|
||||
},
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
for sig, handler in previous.items():
|
||||
signal.signal(sig, handler)
|
||||
if not keep_volumes and not cleanup_failed:
|
||||
remove_runtime(runtime)
|
||||
else:
|
||||
print(
|
||||
"Private runtime retained outside evidence: " + str(runtime), flush=True
|
||||
)
|
||||
return 130 if stop.is_set() else 0
|
||||
|
||||
|
||||
def cleanup_campaign(campaign):
|
||||
runtime = Path(load(campaign / "runtime.json")["private_directory"])
|
||||
if not runtime.exists():
|
||||
raise ExperimentError(
|
||||
"runtime already removed; normal run already removed its containers/volumes"
|
||||
)
|
||||
for case_file in sorted((campaign / "cases").glob("*/case.json")):
|
||||
case = load(case_file)
|
||||
with (case_file.parent / "manual-cleanup.log").open("a") as log:
|
||||
safe_down(campaign, runtime, case["project"], log, False)
|
||||
remove_runtime(runtime)
|
||||
print("Only selected campaign projects removed; evidence kept.")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Reproducible proxy/presigned upload experiment (WSL/Linux, Python stdlib)"
|
||||
)
|
||||
sub = parser.add_subparsers(dest="action", required=True)
|
||||
sub.add_parser("check", help="read-only prerequisites; no build or workload")
|
||||
p = sub.add_parser(
|
||||
"prepare", help="freeze protocol/source/fixtures and build images; no uploads"
|
||||
)
|
||||
p.add_argument("--pilot", action="store_true")
|
||||
p.add_argument("--reuse-images", type=campaign_path)
|
||||
for action in ("run", "summarize", "cleanup"):
|
||||
p = sub.add_parser(action)
|
||||
p.add_argument("campaign", type=campaign_path)
|
||||
if action == "run":
|
||||
p.add_argument("--resume", action="store_true")
|
||||
p.add_argument("--keep-volumes", action="store_true")
|
||||
args = parser.parse_args()
|
||||
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
||||
with (EVIDENCE / ".lock").open("a") as lock:
|
||||
try:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except BlockingIOError:
|
||||
raise SystemExit(
|
||||
"Another experiment command is running; do not run campaigns concurrently."
|
||||
)
|
||||
try:
|
||||
if args.action == "check":
|
||||
info = preflight()
|
||||
print(json.dumps(info, ensure_ascii=False, indent=2))
|
||||
print(
|
||||
"Prerequisites readable, experiment ports free; no uploads executed."
|
||||
)
|
||||
elif args.action == "prepare":
|
||||
prepare(args.pilot, args.reuse_images)
|
||||
elif args.action == "run":
|
||||
raise SystemExit(
|
||||
run_campaign(args.campaign, args.resume, args.keep_volumes)
|
||||
)
|
||||
elif args.action == "cleanup":
|
||||
cleanup_campaign(args.campaign)
|
||||
else:
|
||||
check_seal(args.campaign)
|
||||
result = summarize(args.campaign)
|
||||
print(json.dumps(result["decision"], ensure_ascii=False, indent=2))
|
||||
except (
|
||||
ExperimentError,
|
||||
OSError,
|
||||
subprocess.TimeoutExpired,
|
||||
ValueError,
|
||||
KeyError,
|
||||
) as error:
|
||||
print(
|
||||
"ERROR: "
|
||||
+ (
|
||||
str(error)
|
||||
if isinstance(error, ExperimentError)
|
||||
else type(error).__name__
|
||||
),
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,35 @@
|
||||
import hashlib
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
|
||||
def chunk(kind, payload):
|
||||
return (
|
||||
struct.pack(">I", len(payload))
|
||||
+ kind
|
||||
+ payload
|
||||
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def make_png(width, height, size, seed):
|
||||
compressor = zlib.compressobj(6)
|
||||
compressed = []
|
||||
for y in range(height):
|
||||
color = bytes(((seed + y) % 256, (seed + y * 3) % 256, (seed + y * 7) % 256))
|
||||
compressed.append(compressor.compress(b"\0" + color * width))
|
||||
compressed.append(compressor.flush())
|
||||
header = b"\x89PNG\r\n\x1a\n" + chunk(
|
||||
b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
)
|
||||
image = header + chunk(b"IDAT", b"".join(compressed))
|
||||
# npAD: ancillary, private, reserved uppercase, unsafe-to-copy. The same
|
||||
# decoded pixels occur at both encoded sizes; only this chunk grows.
|
||||
remaining = size - len(image) - len(chunk(b"IEND", b"")) - 12
|
||||
if remaining < 0:
|
||||
raise ValueError("target size cannot hold this PNG")
|
||||
block = hashlib.sha256(str(seed).encode("ascii")).digest()
|
||||
padding = (block * ((remaining + len(block) - 1) // len(block)))[:remaining]
|
||||
result = image + chunk(b"npAD", padding) + chunk(b"IEND", b"")
|
||||
assert len(result) == size
|
||||
return result
|
||||
@@ -0,0 +1,171 @@
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from common import ExperimentError, command
|
||||
|
||||
|
||||
def parse_snapshot(text):
|
||||
sections, current = {}, None
|
||||
for line in text.splitlines():
|
||||
if line.startswith("@"):
|
||||
current = line[1:]
|
||||
sections[current] = []
|
||||
elif current:
|
||||
sections[current].append(line)
|
||||
metrics = {
|
||||
name: int(sections[name][0])
|
||||
for name in ("memory.current", "memory.peak", "memory.max")
|
||||
}
|
||||
metrics["memory.events"] = {
|
||||
row.split()[0]: int(row.split()[1]) for row in sections["memory.events"]
|
||||
}
|
||||
metrics["cpu.stat"] = {
|
||||
row.split()[0]: int(row.split()[1]) for row in sections["cpu.stat"]
|
||||
}
|
||||
interfaces = {}
|
||||
for row in sections["net"]:
|
||||
if ":" not in row:
|
||||
continue
|
||||
name, values = row.split(":", 1)
|
||||
if name.strip() != "lo":
|
||||
values = values.split()
|
||||
interfaces[name.strip()] = {
|
||||
"rx_bytes": int(values[0]),
|
||||
"tx_bytes": int(values[8]),
|
||||
}
|
||||
if not interfaces:
|
||||
raise ExperimentError("no non-loopback interfaces")
|
||||
metrics["interfaces"] = interfaces
|
||||
metrics["rx_bytes"] = sum(i["rx_bytes"] for i in interfaces.values())
|
||||
metrics["tx_bytes"] = sum(i["tx_bytes"] for i in interfaces.values())
|
||||
return metrics
|
||||
|
||||
|
||||
def snapshot(containers):
|
||||
result = {}
|
||||
script = "for item in memory.current memory.peak memory.max memory.events cpu.stat; do printf '@%s\\n' \"$item\"; cat /sys/fs/cgroup/$item || exit 2; done; printf '@net\\n'; cat /proc/net/dev"
|
||||
for item in containers:
|
||||
start = time.monotonic_ns()
|
||||
text = command(["docker", "exec", item["id"], "sh", "-c", script], timeout=15)
|
||||
result[item["id"]] = {
|
||||
**parse_snapshot(text),
|
||||
"read_started_ns": start,
|
||||
"read_finished_ns": time.monotonic_ns(),
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def inspect_containers(ids, project):
|
||||
result = []
|
||||
for item in json.loads(command(["docker", "inspect", *ids])):
|
||||
labels = item["Config"]["Labels"]
|
||||
if labels.get("com.docker.compose.project") != project:
|
||||
raise ExperimentError("container belongs to a different project")
|
||||
result.append(
|
||||
{
|
||||
"id": item["Id"],
|
||||
"hostname": item["Config"]["Hostname"],
|
||||
"service": labels["com.docker.compose.service"],
|
||||
"image": item["Image"],
|
||||
"started_at": item["State"]["StartedAt"],
|
||||
"status": item["State"]["Status"],
|
||||
"oom_killed": item["State"]["OOMKilled"],
|
||||
"restarts": item["RestartCount"],
|
||||
"memory_limit": item["HostConfig"]["Memory"],
|
||||
"nano_cpus": item["HostConfig"]["NanoCpus"],
|
||||
"memory_and_swap_limit": item["HostConfig"]["MemorySwap"],
|
||||
"tmpfs": item["HostConfig"].get("Tmpfs", {}),
|
||||
"volumes": [
|
||||
m.get("Name") for m in item["Mounts"] if m["Type"] == "volume"
|
||||
],
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, containers, path, interval):
|
||||
self.file = path.open("w", encoding="utf-8")
|
||||
self.lock = threading.Lock()
|
||||
self.items, self.threads, self.errors = [], [], []
|
||||
self.counts = {c["id"]: 0 for c in containers}
|
||||
self.stopping = False
|
||||
# The in-container stop marker prevents docker-exec client termination
|
||||
# from leaving a sampling shell running inside a retained container.
|
||||
marker = "/tmp/upload-experiment-sampler.stop"
|
||||
script = 'while [ ! -e "$1" ]; do read current < /sys/fs/cgroup/memory.current || exit 2; read peak < /sys/fs/cgroup/memory.peak || exit 2; read uptime rest < /proc/uptime; printf "%s %s %s\\n" "$current" "$peak" "$uptime"; sleep "$2"; done'
|
||||
for c in containers:
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
c["id"],
|
||||
"sh",
|
||||
"-c",
|
||||
script,
|
||||
"sampler",
|
||||
marker,
|
||||
str(interval),
|
||||
],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
)
|
||||
self.items.append((c, process, marker))
|
||||
thread = threading.Thread(target=self.read, args=(c, process), daemon=True)
|
||||
thread.start()
|
||||
self.threads.append(thread)
|
||||
|
||||
def read(self, container, process):
|
||||
try:
|
||||
for line in process.stdout:
|
||||
current, peak, uptime = line.split()
|
||||
row = {
|
||||
"container": container["id"],
|
||||
"service": container["service"],
|
||||
"monotonic_ns": time.monotonic_ns(),
|
||||
"source_uptime_seconds": float(uptime),
|
||||
"memory_current": int(current),
|
||||
"memory_peak_lifetime": int(peak),
|
||||
}
|
||||
with self.lock:
|
||||
self.counts[container["id"]] += 1
|
||||
self.file.write(json.dumps(row) + "\n")
|
||||
self.file.flush()
|
||||
if not self.stopping:
|
||||
self.errors.append("sampler exited before stop: " + container["id"])
|
||||
except Exception as error:
|
||||
self.errors.append(type(error).__name__)
|
||||
|
||||
def wait_ready(self):
|
||||
deadline = time.monotonic() + 10
|
||||
while not all(count >= 2 for count in self.counts.values()):
|
||||
if self.errors or time.monotonic() > deadline:
|
||||
raise ExperimentError("sampler did not become ready")
|
||||
time.sleep(0.05)
|
||||
|
||||
def close(self):
|
||||
self.stopping = True
|
||||
for c, process, marker in self.items:
|
||||
try:
|
||||
command(["docker", "exec", c["id"], "touch", marker], timeout=3)
|
||||
except Exception:
|
||||
self.errors.append("cannot stop sampler: " + c["id"])
|
||||
try:
|
||||
process.wait(timeout=3)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=2)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
if process.returncode:
|
||||
self.errors.append("sampler exit " + str(process.returncode))
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=3)
|
||||
if thread.is_alive():
|
||||
self.errors.append("sampler reader did not stop")
|
||||
self.file.close()
|
||||
@@ -0,0 +1,416 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import zlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
from common import EXPERIMENT, EVIDENCE, ROOT, save, load, ExperimentError
|
||||
from driver import make_plan, safe_down, validate_compose
|
||||
from fixtures import make_png
|
||||
from telemetry import parse_snapshot
|
||||
from analysis import percentile, decision, case_metrics
|
||||
from client import Client
|
||||
|
||||
|
||||
PROTOCOL = load(EXPERIMENT / "config/protocol.json")
|
||||
|
||||
|
||||
class InstrumentTests(unittest.TestCase):
|
||||
def test_evidence_is_in_repository_root(self):
|
||||
self.assertEqual(EVIDENCE, ROOT / "evidence" / "upload-comparison")
|
||||
|
||||
def test_png_is_valid_deterministic_and_exact_size(self):
|
||||
for width, height, size in [(16, 8, 4096), (32, 16, 8192)]:
|
||||
data = make_png(width, height, size, 123)
|
||||
self.assertEqual(data, make_png(width, height, size, 123))
|
||||
self.assertEqual(len(data), size)
|
||||
self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n")
|
||||
chunks, offset = {}, 8
|
||||
while offset < len(data):
|
||||
length = struct.unpack(">I", data[offset : offset + 4])[0]
|
||||
kind, payload = (
|
||||
data[offset + 4 : offset + 8],
|
||||
data[offset + 8 : offset + 8 + length],
|
||||
)
|
||||
crc = struct.unpack(
|
||||
">I", data[offset + 8 + length : offset + 12 + length]
|
||||
)[0]
|
||||
self.assertEqual(crc, zlib.crc32(kind + payload) & 0xFFFFFFFF)
|
||||
chunks[kind] = payload
|
||||
offset += length + 12
|
||||
self.assertEqual(offset, len(data))
|
||||
self.assertEqual(struct.unpack(">II", chunks[b"IHDR"][:8]), (width, height))
|
||||
self.assertEqual(
|
||||
len(zlib.decompress(chunks[b"IDAT"])), height * (width * 3 + 1)
|
||||
)
|
||||
self.assertIn(b"npAD", chunks)
|
||||
|
||||
def test_all_actual_profile_headers_and_lengths(self):
|
||||
for profile in PROTOCOL["profiles"]:
|
||||
data = make_png(
|
||||
profile["width"],
|
||||
profile["height"],
|
||||
profile["size_bytes"],
|
||||
PROTOCOL["seed"],
|
||||
)
|
||||
self.assertEqual(len(data), profile["size_bytes"])
|
||||
self.assertEqual(
|
||||
struct.unpack(">II", data[16:24]), (profile["width"], profile["height"])
|
||||
)
|
||||
|
||||
def test_plan_has_complete_counterbalanced_pairs(self):
|
||||
plan = make_plan(PROTOCOL)
|
||||
self.assertEqual(len(plan), 24)
|
||||
self.assertEqual(plan, make_plan(PROTOCOL))
|
||||
self.assertEqual(sum(p["mode"] == "proxy" for p in plan[::2]), 6)
|
||||
self.assertEqual(len({p["directory"] for p in plan}), 24)
|
||||
for profile in PROTOCOL["profiles"]:
|
||||
for repetition in range(1, 4):
|
||||
pair = [
|
||||
c["mode"]
|
||||
for c in plan
|
||||
if c["profile"] == profile["name"] and c["repetition"] == repetition
|
||||
]
|
||||
self.assertEqual(sorted(pair), ["direct", "proxy"])
|
||||
changed = copy.deepcopy(PROTOCOL)
|
||||
changed["seed"] += 1
|
||||
self.assertNotEqual(plan, make_plan(changed))
|
||||
|
||||
def test_counters_sum_all_non_loopback_interfaces(self):
|
||||
text = """@memory.current
|
||||
100
|
||||
@memory.peak
|
||||
200
|
||||
@memory.max
|
||||
536870912
|
||||
@memory.events
|
||||
oom 0
|
||||
oom_kill 0
|
||||
max 0
|
||||
@cpu.stat
|
||||
usage_usec 500
|
||||
nr_throttled 1
|
||||
throttled_usec 12
|
||||
@net
|
||||
Inter-| Receive | Transmit
|
||||
lo: 900 0 0 0 0 0 0 0 800 0 0 0 0 0 0 0
|
||||
eth0: 100 0 0 0 0 0 0 0 200 0 0 0 0 0 0 0
|
||||
eth1: 300 0 0 0 0 0 0 0 400 0 0 0 0 0 0 0
|
||||
"""
|
||||
parsed = parse_snapshot(text)
|
||||
self.assertEqual(parsed["rx_bytes"], 400)
|
||||
self.assertEqual(parsed["tx_bytes"], 600)
|
||||
self.assertEqual(parsed["cpu.stat"]["usage_usec"], 500)
|
||||
|
||||
def test_cleanup_cannot_target_another_campaign(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
campaign = Path(folder) / "series-formal-123"
|
||||
campaign.mkdir()
|
||||
save(campaign / "plan.json", make_plan(PROTOCOL))
|
||||
with patch("driver.command") as command:
|
||||
with self.assertRaises(ExperimentError):
|
||||
safe_down(
|
||||
campaign,
|
||||
Path(folder),
|
||||
"hl-upload-exp-another-series-1",
|
||||
None,
|
||||
False,
|
||||
)
|
||||
command.assert_not_called()
|
||||
|
||||
def test_compose_memory_values_may_be_strings(self):
|
||||
services = {}
|
||||
r = PROTOCOL["resources"]
|
||||
for role in ("api", "db", "redis", "s3", "haproxy"):
|
||||
services[role] = {
|
||||
"cpus": r[role + "_cpus"],
|
||||
"mem_limit": str(r[role + "_memory_bytes"]),
|
||||
"memswap_limit": str(r[role + "_memory_bytes"]),
|
||||
}
|
||||
services["api"].update(
|
||||
deploy={"replicas": 2},
|
||||
tmpfs=["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"],
|
||||
environment={
|
||||
"GOMAXPROCS": "1",
|
||||
"GOMEMLIMIT": "384MiB",
|
||||
"DATABASE_MAX_CONNS": "10",
|
||||
},
|
||||
)
|
||||
response = SimpleNamespace(
|
||||
returncode=0, stdout=json.dumps({"services": services})
|
||||
)
|
||||
with patch("driver.subprocess.run", return_value=response):
|
||||
validate_compose(PROTOCOL)
|
||||
services["api"]["memswap_limit"] = "1073741824"
|
||||
response.stdout = json.dumps({"services": services})
|
||||
with patch("driver.subprocess.run", return_value=response):
|
||||
with self.assertRaises(ExperimentError):
|
||||
validate_compose(PROTOCOL)
|
||||
|
||||
|
||||
class DecisionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.plan = make_plan(PROTOCOL)
|
||||
self.cases = [
|
||||
{
|
||||
**c,
|
||||
"api_bytes_per_ready": 1100000 if c["mode"] == "direct" else 2000000,
|
||||
"system_failures": [],
|
||||
"instrumentation_issues": [],
|
||||
}
|
||||
for c in self.plan
|
||||
]
|
||||
|
||||
def result(self, protocol=None):
|
||||
return decision(protocol or PROTOCOL, self.cases, self.plan)
|
||||
|
||||
def test_declared_threshold_can_confirm(self):
|
||||
self.assertEqual(
|
||||
self.result()["criterion_status"], "confirmed_in_tested_conditions"
|
||||
)
|
||||
|
||||
def test_bad_ratio_is_rejected(self):
|
||||
for case in self.cases:
|
||||
if case["mode"] == "direct":
|
||||
case["api_bytes_per_ready"] = 1600000
|
||||
self.assertEqual(self.result()["criterion_status"], "rejected")
|
||||
|
||||
def test_one_bad_pair_cannot_hide_behind_median(self):
|
||||
next(c for c in self.cases if c["mode"] == "direct")["api_bytes_per_ready"] = (
|
||||
1420000
|
||||
)
|
||||
self.assertEqual(self.result()["criterion_status"], "rejected")
|
||||
|
||||
def test_oom_is_not_discarded_when_sampler_also_failed(self):
|
||||
self.cases[0]["system_failures"] = ["oom_event:api"]
|
||||
self.cases[0]["instrumentation_issues"] = ["sampler_error"]
|
||||
self.assertEqual(self.result()["criterion_status"], "rejected")
|
||||
|
||||
def test_broken_instruments_do_not_create_false_traffic_rejection(self):
|
||||
direct = next(c for c in self.cases if c["mode"] == "direct")
|
||||
direct["api_bytes_per_ready"] = 9000000
|
||||
direct["instrumentation_issues"] = ["network_counter_reset:api"]
|
||||
self.assertEqual(self.result()["criterion_status"], "inconclusive")
|
||||
|
||||
def test_missing_case_is_inconclusive(self):
|
||||
self.cases.pop()
|
||||
self.assertEqual(self.result()["criterion_status"], "inconclusive")
|
||||
|
||||
def test_pilot_never_confirms_the_hypothesis(self):
|
||||
protocol = copy.deepcopy(PROTOCOL)
|
||||
protocol["purpose"] = "pilot"
|
||||
self.assertEqual(
|
||||
self.result(protocol)["criterion_status"], "pilot_not_for_hypothesis"
|
||||
)
|
||||
|
||||
def test_percentiles_use_nearest_rank(self):
|
||||
self.assertEqual(percentile(list(range(1, 31)), 0.99), 30)
|
||||
self.assertEqual(percentile(list(range(1, 31)), 0.50), 15)
|
||||
self.assertIsNone(percentile([], 0.50))
|
||||
|
||||
def test_user_interruption_is_not_a_server_failure(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder)
|
||||
save(path / "case.json", {**self.plan[0]})
|
||||
save(path / "outcome.json", {"status": "interrupted"})
|
||||
(path / "requests.jsonl").write_text(
|
||||
json.dumps({"ready": False, "error": "interrupted"}) + "\n"
|
||||
)
|
||||
result = case_metrics(path, PROTOCOL)
|
||||
self.assertEqual(result["errors"], 0)
|
||||
self.assertEqual(result["cancelled"], 1)
|
||||
self.assertEqual(result["system_failures"], [])
|
||||
|
||||
def test_metrics_keep_window_ram_distinct_and_exclude_s3_from_api_traffic(self):
|
||||
protocol = copy.deepcopy(PROTOCOL)
|
||||
protocol.update(clients=2, waves=1, sample_interval_seconds=0.1)
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
path = Path(folder)
|
||||
save(path / "case.json", self.plan[0])
|
||||
save(path / "outcome.json", {"status": "completed"})
|
||||
containers = [
|
||||
{
|
||||
"id": cid,
|
||||
"hostname": cid,
|
||||
"service": role,
|
||||
"status": "running",
|
||||
"started_at": "same",
|
||||
"restarts": 0,
|
||||
"oom_killed": False,
|
||||
"nano_cpus": cpu,
|
||||
}
|
||||
for cid, role, cpu in [
|
||||
("a", "api", 500000000),
|
||||
("b", "api", 500000000),
|
||||
("s", "s3", 1000000000),
|
||||
]
|
||||
]
|
||||
save(path / "containers-before.json", containers)
|
||||
save(path / "containers-after.json", containers)
|
||||
|
||||
def counters(received, sent, ns, peak):
|
||||
return {
|
||||
"rx_bytes": received,
|
||||
"tx_bytes": sent,
|
||||
"memory.current": 100,
|
||||
"memory.peak": peak,
|
||||
"memory.max": 536870912,
|
||||
"memory.events": {"oom": 0, "oom_kill": 0, "max": 0},
|
||||
"cpu.stat": {"usage_usec": ns // 10000},
|
||||
"read_started_ns": ns,
|
||||
"read_finished_ns": ns + 1,
|
||||
}
|
||||
|
||||
ids = [c["id"] for c in containers]
|
||||
m = {
|
||||
"started_ns": 10000000000,
|
||||
"finished_ns": 12000000000,
|
||||
"generator_cpu_seconds": 0.1,
|
||||
"idle_before": {cid: counters(0, 0, 0, 300) for cid in ids},
|
||||
"idle_after": {cid: counters(0, 0, 5000000000, 300) for cid in ids},
|
||||
"before": {cid: counters(0, 0, 10000000000, 300) for cid in ids},
|
||||
"after": {
|
||||
cid: counters(4000000, 4000000, 12000000000, 400) for cid in ids
|
||||
},
|
||||
}
|
||||
save(path / "measurement.json", m)
|
||||
requests = [
|
||||
{
|
||||
"wave": 0,
|
||||
"ready": True,
|
||||
"started_ns": 10100000000 + i * 1000000,
|
||||
"finished_ns": 11900000000,
|
||||
"attachment": {"id": cid, "instance": cid},
|
||||
}
|
||||
for i, cid in enumerate(("a", "b"))
|
||||
]
|
||||
(path / "requests.jsonl").write_text(
|
||||
"".join(json.dumps(r) + "\n" for r in requests)
|
||||
)
|
||||
(path / "warmup.jsonl").write_text(
|
||||
"".join(json.dumps(r) + "\n" for r in requests)
|
||||
)
|
||||
(path / "verification.jsonl").write_text(
|
||||
"".join(
|
||||
json.dumps({"id": cid, "verified": True}) + "\n"
|
||||
for cid in ("a", "b")
|
||||
)
|
||||
)
|
||||
(path / "samples.jsonl").write_text(
|
||||
"".join(
|
||||
json.dumps(
|
||||
{
|
||||
"container": cid,
|
||||
"memory_current": 200,
|
||||
"monotonic_ns": 10000000000 + i * 100000000,
|
||||
}
|
||||
)
|
||||
+ "\n"
|
||||
for cid in ids
|
||||
for i in range(21)
|
||||
)
|
||||
)
|
||||
result = case_metrics(path, protocol)
|
||||
self.assertEqual(result["api_rx_tx_bytes"], 16000000)
|
||||
self.assertEqual(result["api_bytes_per_ready"], 8000000)
|
||||
self.assertEqual(result["containers"][0]["sampled_peak_window"], 200)
|
||||
self.assertEqual(result["containers"][0]["peak_lifetime_after"], 400)
|
||||
self.assertEqual(result["system_failures"], [])
|
||||
self.assertEqual(result["instrumentation_issues"], [])
|
||||
|
||||
|
||||
class TransportTests(unittest.TestCase):
|
||||
def test_direct_stages_do_not_send_bearer_to_s3_or_leak_secrets_to_records(self):
|
||||
observed, data = [], b"synthetic PNG transport fixture"
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
token, signed_field = "DO-NOT-PERSIST-TOKEN", "DO-NOT-PERSIST-SIGNATURE"
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
observed.append(
|
||||
{
|
||||
"path": self.path,
|
||||
"authorization": self.headers.get("Authorization"),
|
||||
"body": body,
|
||||
}
|
||||
)
|
||||
if self.path.endswith("/attachment-uploads"):
|
||||
response = {
|
||||
"upload_id": "upload",
|
||||
"upload": {
|
||||
"url": f"http://127.0.0.1:{self.server.server_port}/bucket",
|
||||
"fields": {
|
||||
"key": "staging/direct/upload",
|
||||
"policy": signed_field,
|
||||
},
|
||||
},
|
||||
}
|
||||
elif self.path == "/bucket":
|
||||
self.send_response(204)
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
else:
|
||||
response = {
|
||||
"id": "attachment",
|
||||
"status": "ready",
|
||||
"size_bytes": len(data),
|
||||
"sha256": checksum,
|
||||
}
|
||||
encoded = json.dumps(response).encode()
|
||||
self.send_response(201)
|
||||
self.send_header("Content-Length", str(len(encoded)))
|
||||
self.send_header("X-Instance-Id", "api-test")
|
||||
self.end_headers()
|
||||
self.wfile.write(encoded)
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
protocol = copy.deepcopy(PROTOCOL)
|
||||
protocol["ports"] = {"api": server.server_port, "s3": server.server_port}
|
||||
client = Client(
|
||||
f"http://127.0.0.1:{server.server_port}", protocol, threading.Event()
|
||||
)
|
||||
records = []
|
||||
client.upload(
|
||||
"direct",
|
||||
{"token": token},
|
||||
"chat",
|
||||
data,
|
||||
checksum,
|
||||
records,
|
||||
__import__("time").monotonic() + 10,
|
||||
)
|
||||
self.assertEqual(
|
||||
[r["stage"] for r in records],
|
||||
["client_sha256", "init", "s3_post", "complete"],
|
||||
)
|
||||
self.assertEqual(observed[0]["authorization"], "Bearer " + token)
|
||||
self.assertIsNone(observed[1]["authorization"])
|
||||
self.assertIn(data, observed[1]["body"])
|
||||
self.assertEqual(observed[2]["authorization"], "Bearer " + token)
|
||||
self.assertNotIn(token, json.dumps(records))
|
||||
self.assertNotIn(signed_field, json.dumps(records))
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,213 @@
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
|
||||
import driver
|
||||
from analysis import json_lines
|
||||
from client import FlowError
|
||||
from common import (
|
||||
ROOT,
|
||||
ExperimentError,
|
||||
check_seal,
|
||||
digest,
|
||||
load,
|
||||
save,
|
||||
source_manifest,
|
||||
)
|
||||
|
||||
|
||||
class OrchestrationTests(unittest.TestCase):
|
||||
def test_source_snapshot_excludes_secrets_evidence_and_git(self):
|
||||
manifest = source_manifest()
|
||||
self.assertIn("src/backend/services/direct_upload.go", manifest)
|
||||
self.assertIn("experiments/docs/HYPOTHESIS.md", manifest)
|
||||
for name in manifest:
|
||||
self.assertNotIn(
|
||||
name.split("/")[0], (".git", ".memory", "secrets", "evidence")
|
||||
)
|
||||
self.assertNotEqual(Path(name).name, ".env")
|
||||
|
||||
def test_prepare_seals_protocol_before_builds_and_does_not_upload(self):
|
||||
with tempfile.TemporaryDirectory(dir=ROOT) as folder:
|
||||
evidence = Path(folder)
|
||||
names = [
|
||||
"src/go.mod",
|
||||
"infra/minio.Dockerfile",
|
||||
"experiments/src/analysis.py",
|
||||
"experiments/src/common.py",
|
||||
]
|
||||
manifest = {name: digest(ROOT / name) for name in names}
|
||||
|
||||
def fake_command(args, *_args, **_kwargs):
|
||||
if args[:2] == ["docker", "build"]:
|
||||
campaign = next(evidence.iterdir())
|
||||
self.assertTrue((campaign / "protocol.json").exists())
|
||||
self.assertFalse((campaign / "cases").exists())
|
||||
return ""
|
||||
if args[:3] == ["docker", "image", "inspect"]:
|
||||
return json.dumps(
|
||||
[
|
||||
{
|
||||
"Id": "sha256:" + "a" * 64,
|
||||
"RepoDigests": [],
|
||||
"Os": "linux",
|
||||
"Architecture": "amd64",
|
||||
"RootFS": {"Layers": ["layer"]},
|
||||
}
|
||||
]
|
||||
)
|
||||
if args[0] == "git":
|
||||
return "synthetic-git-metadata"
|
||||
raise AssertionError("unexpected command in prepare")
|
||||
|
||||
with (
|
||||
patch("driver.EVIDENCE", evidence),
|
||||
patch("driver.preflight", return_value={"purpose": "fake-host"}),
|
||||
patch("driver.source_manifest", return_value=manifest),
|
||||
patch("driver.check_sources"),
|
||||
patch("driver.command", side_effect=fake_command),
|
||||
patch("driver.Client") as client,
|
||||
patch("builtins.print"),
|
||||
):
|
||||
campaign = driver.prepare(pilot=True)
|
||||
client.assert_not_called()
|
||||
check_seal(campaign)
|
||||
self.assertEqual(load(campaign / "prepared.json")["cases"], 4)
|
||||
self.assertEqual(load(campaign / "protocol.json")["purpose"], "pilot")
|
||||
self.assertFalse((campaign / "cases").exists())
|
||||
protocol = load(campaign / "protocol.json")
|
||||
protocol["clients"] += 1
|
||||
save(campaign / "protocol.json", protocol)
|
||||
with self.assertRaises(ExperimentError):
|
||||
check_seal(campaign)
|
||||
|
||||
def test_failed_uploads_are_saved_and_verification_starts_after_counters_close(
|
||||
self,
|
||||
):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
campaign = Path(folder)
|
||||
protocol = load(driver.EXPERIMENT / "config/protocol.json")
|
||||
protocol.update(
|
||||
clients=2,
|
||||
waves=1,
|
||||
warmup_clients=2,
|
||||
settle_seconds=0,
|
||||
idle_seconds=0.001,
|
||||
)
|
||||
save(campaign / "protocol.json", protocol)
|
||||
save(campaign / "images.json", {"app": {"id": "app"}, "s3": {"id": "s3"}})
|
||||
data = b"synthetic-fixture"
|
||||
checksum = hashlib.sha256(data).hexdigest()
|
||||
save(
|
||||
campaign / "fixtures.json",
|
||||
[{"name": "fake", "filename": "fake.png", "sha256": checksum}],
|
||||
)
|
||||
(campaign / "fixtures").mkdir()
|
||||
(campaign / "fixtures/fake.png").write_bytes(data)
|
||||
case = {
|
||||
"directory": "01-fake",
|
||||
"number": 1,
|
||||
"profile": "fake",
|
||||
"mode": "direct",
|
||||
"size_bytes": len(data),
|
||||
"repetition": 1,
|
||||
}
|
||||
containers = [
|
||||
{
|
||||
"id": cid,
|
||||
"hostname": cid,
|
||||
"service": role,
|
||||
"image": image,
|
||||
"memory_limit": 536870912,
|
||||
"memory_and_swap_limit": 536870912,
|
||||
"nano_cpus": 500000000,
|
||||
"tmpfs": {"/tmp/uploads": "rw,noexec,nosuid,size=16m,mode=1777"},
|
||||
}
|
||||
for cid, role, image in [
|
||||
("a", "api", "app"),
|
||||
("b", "api", "app"),
|
||||
("s", "s3", "s3"),
|
||||
]
|
||||
]
|
||||
state = {"snapshots": 0, "sampler_closed": False, "uploads": 0}
|
||||
mutex = threading.Lock()
|
||||
|
||||
class FakeClient:
|
||||
def __init__(self, _base, p, _stop):
|
||||
self.protocol = p
|
||||
|
||||
def bootstrap(self):
|
||||
return [{"token": "secret-a"}, {"token": "secret-b"}], "chat"
|
||||
|
||||
def upload(
|
||||
self, _mode, _actor, _chat, _data, _digest, records, _deadline
|
||||
):
|
||||
with mutex:
|
||||
state["uploads"] += 1
|
||||
index = state["uploads"]
|
||||
if index == 3:
|
||||
raise FlowError("http_503")
|
||||
return {
|
||||
"id": "attachment-" + str(index),
|
||||
"instance": "a" if index % 2 else "b",
|
||||
}
|
||||
|
||||
def verify(self, _actor, attachment, _size, _digest):
|
||||
if state["snapshots"] != 4 or not state["sampler_closed"]:
|
||||
raise AssertionError("verification contaminated measurement")
|
||||
return {"id": attachment["id"], "verified": True}
|
||||
|
||||
class FakeSampler:
|
||||
errors = []
|
||||
|
||||
def __init__(self, _containers, path, _interval):
|
||||
path.write_text("")
|
||||
|
||||
def wait_ready(self):
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
state["sampler_closed"] = True
|
||||
|
||||
def fake_snapshot(_containers):
|
||||
state["snapshots"] += 1
|
||||
return {c["id"]: {"sample": "fake"} for c in containers}
|
||||
|
||||
def fake_command(args, *_args, **_kwargs):
|
||||
return "a\nb\ns" if "ps" in args else ""
|
||||
|
||||
with (
|
||||
patch("driver.command", side_effect=fake_command),
|
||||
patch("driver.inspect_containers", return_value=containers),
|
||||
patch("driver.Client", FakeClient),
|
||||
patch("driver.Sampler", FakeSampler),
|
||||
patch("driver.snapshot", side_effect=fake_snapshot),
|
||||
):
|
||||
outcome = driver.run_case(
|
||||
campaign,
|
||||
case,
|
||||
Path(folder),
|
||||
"hl-upload-exp-fake-1",
|
||||
threading.Event(),
|
||||
)
|
||||
self.assertEqual(outcome["status"], "completed")
|
||||
directory = campaign / "cases/01-fake"
|
||||
requests = json_lines(directory / "requests.jsonl")
|
||||
self.assertEqual(len(requests), 2)
|
||||
self.assertEqual(sum(r["ready"] for r in requests), 1)
|
||||
self.assertEqual(
|
||||
next(r for r in requests if not r["ready"])["error"], "http_503"
|
||||
)
|
||||
self.assertEqual(len(json_lines(directory / "verification.jsonl")), 1)
|
||||
self.assertNotIn("secret-a", (directory / "requests.jsonl").read_text())
|
||||
self.assertTrue((directory / "measurement.json").exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Offline plot checks; no application uploads or research measurements."""
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
AVAILABLE = all(importlib.util.find_spec(name) for name in ("pandas", "matplotlib"))
|
||||
|
||||
|
||||
@unittest.skipUnless(AVAILABLE, "plot dependencies: run bash experiments/setup.sh")
|
||||
class PlotTests(unittest.TestCase):
|
||||
def test_plots_preserve_inputs_and_keep_all_attempts(self):
|
||||
experiment = Path(__file__).resolve().parents[1]
|
||||
spec = importlib.util.spec_from_file_location("experiment_plot", experiment / "plot.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
campaign = Path(temporary)
|
||||
protocol = {"profiles": [{"name": "sample"}], "criteria": {
|
||||
"median_direct_proxy_traffic_ratio_max": 0.65,
|
||||
"each_pair_traffic_ratio_max": 0.70}}
|
||||
summary = {"decision": {"criterion_status": "inconclusive", "pairs": [
|
||||
{"profile": "sample", "repetition": 1, "direct_proxy_ratio": 0.6,
|
||||
"instrumentation_valid": False}]}}
|
||||
(campaign / "protocol.json").write_text(json.dumps(protocol))
|
||||
(campaign / "metrics.json").write_text(json.dumps(summary))
|
||||
(campaign / "metrics.csv").write_text(
|
||||
"profile,mode,repetition,errors,api_bytes_per_ready,"
|
||||
"api_sampled_peak_window_max,api_peak_lifetime_max,p95_seconds\n"
|
||||
"sample,proxy,1,1,2000000,200000000,300000000,2\n"
|
||||
"sample,direct,1,0,1200000,210000000,310000000,3\n"
|
||||
"sample,direct,2,2,,,,\n")
|
||||
before = {f.name: f.read_bytes() for f in campaign.glob("*.json")}
|
||||
before["metrics.csv"] = (campaign / "metrics.csv").read_bytes()
|
||||
output = module.create_plots(campaign)
|
||||
self.assertEqual(len(list(output.glob("*.png"))), 6)
|
||||
self.assertEqual(len(list(output.glob("*.pdf"))), 0)
|
||||
self.assertTrue(all(f.stat().st_size > 1000 for f in output.glob("*.png")))
|
||||
provenance = json.loads((output / "provenance.json").read_text())
|
||||
self.assertEqual(provenance["criterion_status"], "inconclusive")
|
||||
for name, content in before.items():
|
||||
self.assertEqual((campaign / name).read_bytes(), content)
|
||||
self.assertNotEqual(output, module.create_plots(campaign))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user