feat: plan the experiment

This commit is contained in:
TAskMAster339
2026-10-18 23:14:01 +03:00
parent d7e7ea93f1
commit 989280a763
20 changed files with 3074 additions and 0 deletions
+23
View File
@@ -0,0 +1,23 @@
# Эксперимент: proxy upload и presigned POST
Сравниваем трафик API, память и время загрузки. Результаты всех серий — в корневом [evidence/upload-comparison](../evidence/upload-comparison/).
## Документация
- [Гипотеза и критерии](docs/HYPOTHESIS.md)
- [Методика и метрики](docs/METHODOLOGY.md)
- [Ограничения](docs/LIMITATIONS.md)
- [Запуск, графики и повтор](docs/RUNBOOK.md)
## Структура
| Каталог / файл | Назначение |
| --- | --- |
| [config/protocol.json](config/protocol.json) | Параметры и пороги эксперимента |
| [config/compose.yml](config/compose.yml) | Отдельный стенд |
| [src](src/) | Генератор данных, нагрузка и измерители |
| [tests](tests/) | Проверки инструментов |
| [plot.py](plot.py) | Графики из результатов серии |
| [setup.sh](setup.sh) | Создание корневого `.venv` для анализа |
Начни с [инструкции запуска](docs/RUNBOOK.md): проверка окружения → пилот → основная серия → графики. Пилот проверяет инструменты и не подтверждает гипотезу. Измерения пока не выполнены.
+157
View File
@@ -0,0 +1,157 @@
services:
db:
image: ${DB_IMAGE}
pull_policy: never
cpus: 1
mem_limit: 512m
memswap_limit: 512m
networks: [data]
environment:
POSTGRES_DB: messenger_exp
POSTGRES_USER: messenger_exp
POSTGRES_PASSWORD_FILE: /run/secrets/db_password
volumes:
- postgres_data:/var/lib/postgresql/data
- ${SECRET_DIR}/db_password:/run/secrets/db_password:ro
healthcheck:
test: [CMD-SHELL, pg_isready -U messenger_exp -d messenger_exp]
interval: 2s
timeout: 2s
retries: 30
redis:
image: ${REDIS_IMAGE}
pull_policy: never
cpus: 0.25
mem_limit: 128m
memswap_limit: 128m
networks: [data]
command: [redis-server, --appendonly, "yes", --appendfsync, everysec, --dir, /data]
volumes: [redis_data:/data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
s3:
image: ${S3_IMAGE}
pull_policy: never
cpus: 1
mem_limit: 512m
memswap_limit: 512m
networks: [edge, data]
ports: ["127.0.0.1:${S3_PORT}:9000"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/root_access
MINIO_ROOT_PASSWORD_FILE: /run/secrets/root_secret
MINIO_BROWSER: "off"
MINIO_API_CORS_ALLOW_ORIGIN: http://127.0.0.1:5173,http://localhost:5173
volumes:
- s3_data:/data
- ${SECRET_DIR}/root_access:/run/secrets/root_access:ro
- ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro
healthcheck:
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/ready]
interval: 2s
timeout: 2s
retries: 30
migrate:
image: ${APP_IMAGE}
pull_policy: never
cpus: 0.5
mem_limit: 128m
networks: [data]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
volumes:
- ${SECRET_DIR}/database_url:/run/secrets/database_url:ro
command: [./messenger, migrate, up]
depends_on:
db: {condition: service_healthy}
s3-bootstrap:
image: ${APP_IMAGE}
pull_policy: never
cpus: 0.5
mem_limit: 128m
networks: [data]
environment:
<<: &s3
S3_ENDPOINT: http://s3:9000
S3_PUBLIC_ENDPOINT: http://127.0.0.1:${S3_PORT}
S3_REGION: us-east-1
S3_BUCKET: messenger-experiment
S3_ACCESS_KEY_ID_FILE: /run/secrets/api_access
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/api_secret
S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/root_access
S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/root_secret
volumes:
- ${SECRET_DIR}/api_access:/run/secrets/api_access:ro
- ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro
- ${SECRET_DIR}/root_access:/run/secrets/root_access:ro
- ${SECRET_DIR}/root_secret:/run/secrets/root_secret:ro
command: [./messenger, s3-bootstrap]
depends_on:
s3: {condition: service_healthy}
api:
image: ${APP_IMAGE}
pull_policy: never
cpus: 0.5
mem_limit: 512m
memswap_limit: 512m
deploy: {replicas: 2}
networks: [proxy, data]
stop_grace_period: 15s
environment:
<<: *s3
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: experiment
GOMAXPROCS: "1"
GOMEMLIMIT: 384MiB
DATABASE_MAX_CONNS: "10"
SESSION_TTL_SECONDS: "3600"
IDEMPOTENCY_TTL_SECONDS: "86400"
UPLOAD_TMP_DIR: /tmp/uploads
UPLOAD_CLEANUP_GRACE_SECONDS: "600"
volumes:
- ${SECRET_DIR}/api_access:/run/secrets/api_access:ro
- ${SECRET_DIR}/api_secret:/run/secrets/api_secret:ro
- ${SECRET_DIR}/database_url:/run/secrets/database_url:ro
- ${SECRET_DIR}/cache_url:/run/secrets/cache_url:ro
tmpfs: ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"]
depends_on:
migrate: {condition: service_completed_successfully}
s3-bootstrap: {condition: service_completed_successfully}
redis: {condition: service_healthy}
healthcheck:
test: [CMD, ./messenger, healthcheck]
interval: 2s
timeout: 3s
retries: 30
haproxy:
image: ${HAPROXY_IMAGE}
pull_policy: never
cpus: 0.5
mem_limit: 128m
memswap_limit: 128m
networks: [edge, proxy]
environment: {API_REPLICAS: "2", APP_PORT: "8080"}
ports: ["127.0.0.1:${API_PORT}:8080", "127.0.0.1:${STATS_PORT}:8404"]
volumes:
- ${REPO_ROOT}/infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 2s
timeout: 3s
retries: 30
volumes:
postgres_data:
redis_data:
s3_data:
networks:
edge:
proxy: {internal: true}
data: {internal: true}
+53
View File
@@ -0,0 +1,53 @@
{
"version": 1,
"name": "proxy-versus-presigned-post",
"purpose": "confirmatory",
"hypothesis": "Presigned POST uses at most 65% of proxy API RX+TX bytes per ready, byte-verified attachment in each profile, without request errors, OOM or API restarts.",
"seed": 20261018,
"repetitions": 3,
"clients": 10,
"waves": 3,
"wave_pause_seconds": 1,
"warmup_clients": 4,
"settle_seconds": 2,
"idle_seconds": 5,
"sample_interval_seconds": 0.1,
"request_timeout_seconds": 35,
"flow_timeout_seconds": 90,
"ports": {"api": 18080, "s3": 19000, "stats": 18404},
"resources": {
"api_replicas": 2,
"api_cpus": 0.5,
"api_memory_bytes": 536870912,
"api_tmpfs_bytes": 16777216,
"api_go_memory_limit": "384MiB",
"api_gomaxprocs": 1,
"database_max_connections_per_replica": 10,
"db_cpus": 1,
"db_memory_bytes": 536870912,
"s3_cpus": 1,
"s3_memory_bytes": 536870912,
"redis_cpus": 0.25,
"redis_memory_bytes": 134217728,
"haproxy_cpus": 0.5,
"haproxy_memory_bytes": 134217728
},
"images": {"db": "postgres:16.10-bookworm", "redis": "redis:7.4.11-bookworm", "haproxy": "haproxy:3.2.25-alpine"},
"profiles": [
{"name": "small-moderate", "size_bytes": 1048576, "width": 1024, "height": 1024},
{"name": "large-moderate", "size_bytes": 10485760, "width": 1024, "height": 1024},
{"name": "small-pixels40m", "size_bytes": 1048576, "width": 8000, "height": 5000},
{"name": "large-pixels40m", "size_bytes": 10485760, "width": 8000, "height": 5000}
],
"criteria": {
"median_direct_proxy_traffic_ratio_max": 0.65,
"each_pair_traffic_ratio_max": 0.70,
"errors_allowed": 0,
"api_peak_strictly_below_bytes": 536870912,
"start_spread_seconds_max": 0.1,
"idle_background_payload_fraction_max": 0.05,
"sampler_coverage_fraction_min": 0.7,
"sampler_gap_seconds_max": 1.0,
"generator_cpu_cores_max": 0.8
}
}
+44
View File
@@ -0,0 +1,44 @@
# Гипотеза, версия 1
[README](../README.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md)
**H1:** presigned POST уменьшает суммарный RX+TX двух API на готовое, побайтно проверенное вложение минимум на 35% по медиане парных повторов каждого профиля. Ошибок, OOM и рестартов быть не должно.
Proxy принимает файл и передаёт его S3. Direct отправляет файл сразу в S3, но API скачивает конечный объект для проверки. Поэтому ожидается снижение трафика API; снижение RAM не обещается.
## Показатель и критерии
```text
b = сумма RX+TX двух API за окно / число ready-вложений
R = b_direct / b_proxy для одного профиля и повтора
```
Все ready-вложения затем проверяются по длине и SHA-256. Фон измеряется отдельно и не вычитается.
Для подтверждения нужны все условия:
- По каждому из 4 профилей есть 3 полные пары: медиана R ≤ 0.65, каждый R ≤ 0.70.
- Все 720 измеряемых загрузок и 96 загрузок прогрева успешны; оригиналы измеряемых загрузок совпали.
- Нет OOM, рестартов и завершений API/S3; lifetime peak каждой API строго ниже 512 МиБ.
- Обе API участвовали в прогреве и измерении.
## Пригодность измерений
| Проверка | Порог |
| --- | --- |
| Разброс старта клиентов в волне | ≤100 мс |
| Покрытие RAM-точками с шагом 100 мс | ≥70% |
| Максимальный промежуток между точками, включая границы окна | ≤1 с |
| Оценённый фон API относительно payload прогона | ≤5% |
| Средний CPU генератора | <0.8 ядра |
Нужны полные записи запросов, проверок и счётчиков без необъяснённых сбросов.
## Решение
- `confirmed_in_tested_conditions` — все критерии выполнены.
- `rejected` — ошибки системы/оригиналов, OOM/рестарт/предел памяти либо превышение порогов трафика на пригодных данных.
- `inconclusive` — данных недостаточно или измерители/запуск нарушены, системного опровержения нет.
- `pilot_not_for_hypothesis` — пилот, независимо от чисел.
Ошибка измерителя не отменяет известный OOM или отказ приложения. Отмена пользователем не считается отказом приложения. Все попытки сохраняются. Пороги фиксируются до запуска; изменение требует новой версии протокола и новой серии.
+16
View File
@@ -0,0 +1,16 @@
# Ограничения
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Запуск](RUNBOOK.md)
- **Один WSL/Docker-host.** Сервисы и генератор делят ресурсы. Повторы не являются независимыми машинами; посторонняя нагрузка и page cache могут влиять на результат.
- **Локальный Python-клиент, без TLS.** Новое TCP-соединение на каждой стадии. Результат не переносится напрямую на WAN, браузер и React/Web Crypto.
- **Только синтетический RGB8 PNG.** Дополнительный chunk управляет encoded-размером. Это не модель JPEG-фотографий или всех допустимых изображений.
- **Короткие волны нагрузки.** 10 клиентов, максимум два декодера. Эксперимент не определяет максимальный RPS или длительную устойчивость. Три пары дают ограниченную оценку разброса; p99 из 30 запросов близок к максимуму.
- **Direct не устраняет стоимость обработки.** API читает и декодирует final; S3 делает COPY и хранит staging. Снижение трафика API не означает снижение общей стоимости системы. Дисковый I/O отдельно не исследуется.
- **RAM-пики различаются.** Sampled max может пропустить короткий всплеск; kernel lifetime peak включает startup и прогрев. tmpfs и sampler входят в память cgroup; GOMEMLIMIT не равен всей памяти контейнера.
- **Измерители создают нагрузку.** Sampler, хеширование и запись evidence расходуют CPU/RAM. Порог CPU генератора лишь сигнал возможного ограничения. Сетевые счётчики включают SQL, Redis и healthcheck; фон не вычитается.
- **Повтор зависит от артефактов.** Нужны сохранённые fixtures, исходники и image ID. Пересборка после удаления образов может дать другую базу; версии хоста, Python и zlib фиксируются.
- **Отказы не скрываются.** Все попытки сохраняются. Ошибки/OOM опровергают работоспособность; пропуски измерителя могут сделать результат неопределённым. Исправление проверяется новой серией.
- **Без fault/security-нагрузки.** ACL, подписи и компенсация проверяются отдельно. Cleanup worker исключён из экспериментального стенда, чтобы не менять измеряемый трафик.
До фактических измерений и проверки качества данных выводов о выигрыше нет.
+56
View File
@@ -0,0 +1,56 @@
# Методика, версия 1
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Ограничения](LIMITATIONS.md) · [Запуск](RUNBOOK.md)
## Сравнение
- **Proxy:** binary POST в API до `201 ready`.
- **Direct:** SHA-256 на клиенте → init API → presigned POST в S3 → complete API до `201 ready`.
Повторов запросов нет. Обе ветки используют один образ API и полную проверку изображения. Direct включает COPY внутри S3 и чтение конечного объекта API.
## Стенд и данные
| Сервис | CPU | RAM |
| --- | --- | --- |
| API ×2 | по 0.5 | по 512 МиБ |
| PostgreSQL / MinIO | по 1 | по 512 МиБ |
| Redis | 0.25 | 128 МиБ |
| HAProxy | 0.5 | 128 МиБ |
Каждая API: один декодер, tmpfs 16 МиБ, GOMAXPROCS=1, GOMEMLIMIT=384MiB, пул БД=10. Swap отключён. Таймауты: операция API 25 с, HTTP 30 с, HAProxy/socket 35 с; бюджет flow 90 с проверяется между стадиями.
Четыре RGB8 PNG-профиля: **1 и 10 МиБ × 1024×1024 и 8000×5000 пикселей**. Размер доводится корректным ancillary chunk `npAD` перед IEND; в паре размеров одной геометрии пиксели одинаковые. Файлы и SHA-256 сохраняются.
## Порядок
Seed `20261018`, 3 парных повтора каждого профиля, порядок пар перемешан; 6 пар proxy→direct и 6 direct→proxy. Всего 24 прогона, 720 измеряемых загрузок.
Для каждого прогона:
1. Поднять отдельный Compose-проект со свежими volumes и закреплёнными image ID. Создать двух пользователей и чат.
2. Выполнить 4 параллельных загрузки прогрева, проверить метаданные; подождать 2 с.
3. Запустить sampler API/S3 с шагом 100 мс. Измерить фон за 5 с.
4. Снять начальные счётчики. Выполнить 3 волны по 10 клиентов со стартовым барьером и паузой 1 с между волнами.
5. Снять конечные счётчики и остановить sampler. Затем скачать оригиналы вторым участником и проверить длину, MIME и SHA-256.
6. Сохранить состояния и ошибки, остановить только этот проект. По умолчанию удалить его volumes.
Файл заранее находится в RAM клиента; direct checksum входит во время flow. Исходники не меняются, параллельная нагрузка запрещена. Ошибки запросов сохраняются, серия продолжается; отмена, изменение исходников или сбой окружения/уборки останавливают запуск.
## Метрики и сохранение
| Метрика | Источник / смысл |
| --- | --- |
| API RX+TX | Разность `/proc/net/dev` всех интерфейсов кроме lo, сумма двух API; служебный обмен включён |
| S3 RX+TX | Отдельный показатель, не складывается с API |
| RAM в окне | Максимум sampled `memory.current`, baseline и интервалы точек |
| Lifetime RAM | `memory.peak` до/после, включая startup и прогрев |
| CPU / throttling / OOM | `cpu.stat`, `memory.events`, состояния и рестарты контейнеров |
| Время / goodput | Monotonic clock; стадии, p50/p95/p99 успешных flow, ready/сек за окно |
| Генератор | CPU, RSS, фактический разброс старта клиентов |
`prepare` фиксирует протокол, план, исходники, fixtures и image ID в `seal.json`. Перед прогонами проверяются контрольные суммы. `summarize` создаёт `metrics.json`, `metrics.csv` и provenance; сравнивает парные R и их медиану отдельно по профилям.
В корневом `evidence/upload-comparison/<серия>/`: замороженная конфигурация и исходники, `cases/*` с запросами, RAM-точками, счётчиками, проверками файлов и ошибками. Секреты и подписанные формы не записываются. Временные credentials находятся вне evidence в `/tmp/highload-upload-exp-*`.
Источники измерителей: [сетевая статистика Linux](https://www.kernel.org/doc/html/latest/networking/statistics.html), [cgroup v2](https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html).
+80
View File
@@ -0,0 +1,80 @@
# Запуск и повтор
[README](../README.md) · [Гипотеза](HYPOTHESIS.md) · [Методика](METHODOLOGY.md) · [Ограничения](LIMITATIONS.md)
## Подготовка
Команды выполнять в **WSL/Linux из корня лабораторной**. Для закреплённого окружения анализа нужны Python 3.12, модуль `venv`, Docker с cgroup v2/`memory.peak` и Compose v2+. Ориентир: 4 CPU, 6 ГиБ RAM Docker, 10 ГиБ диска. Порты 18080, 19000, 18404 должны быть свободны.
```sh
bash experiments/setup.sh
.venv/bin/python -m unittest discover -s experiments/tests -v
.venv/bin/python experiments/run.py check
```
`setup.sh` создаёт `.venv` в корне и устанавливает pandas/matplotlib из [requirements.txt](../requirements.txt). Версии прямых и транзитивных зависимостей закреплены. Они нужны только для графиков; runner использует стандартную библиотеку (Python 3.9+). `.venv` уже исключён из Git. Если `venv` недоступен: `sudo apt install python3-venv`.
Не менять исходники между prepare и концом серии, не запускать параллельно сборки/тесты/нагрузку. Рабочий стенд не останавливается автоматически; оставь его без нагрузки.
## Пилот
```sh
.venv/bin/python experiments/run.py prepare --pilot
# Подставь путь из строки Prepared:
PILOT=evidence/upload-comparison/<каталог-pilot>
.venv/bin/python experiments/run.py run "$PILOT"
.venv/bin/python experiments/run.py summarize "$PILOT"
.venv/bin/python experiments/plot.py "$PILOT"
```
Пилот: 2 moderate-профиля, 2 клиента, 1 волна, 1 парный повтор — 4 прогона и 8 измеряемых загрузок. Он проверяет инструменты и не участвует в H1. При исправлении инструментов сохрани старые данные и подготовь новую серию.
## Основная серия
```sh
.venv/bin/python experiments/run.py prepare --reuse-images "$PILOT"
CAMPAIGN=evidence/upload-comparison/<каталог-formal>
.venv/bin/python experiments/run.py run "$CAMPAIGN"
.venv/bin/python experiments/run.py summarize "$CAMPAIGN"
.venv/bin/python experiments/plot.py "$CAMPAIGN"
```
При изменении исходников или без пилота используй `prepare` без `--reuse-images`. Подготовка собирает собственные образы и фиксирует их ID, но не выполняет загрузки. Основная серия: 24 прогона, 720 измеряемых загрузок.
`run`: код 0 — план завершён, 1 — сбой окружения/оркестратора, 130 — отмена. Решение о H1 находится в `metrics.json → decision`, а не в коде выхода.
## Результаты
Весь каталог серии сохраняется в **корневом `evidence/upload-comparison/`**:
- `protocol.json`, `plan.json`, `seal.json`, исходники, fixtures и image ID — условия повторения.
- `cases/*` — сырые запросы, RAM-точки, счётчики, проверки оригиналов, состояния и ошибки.
- `metrics.json` / `metrics.csv` — итоговые показатели и критерии; `analysis-provenance.json` — версия анализа.
- `plots/<время>/` — PNG-графики трафика, отношения direct/proxy, RAM, p95 и ошибок; `provenance.json` фиксирует входы и версии библиотек.
Графики показывают все доступные прогоны; пропуски не заменяются нулями. Качество измерений и итог H1 проверяй по JSON. Для разбора передай весь каталог, включая неудачные прогоны. `/tmp/highload-upload-exp-*` содержит credentials: его не архивировать.
## Повтор, остановка и уборка
Новый полный повтор без перезаписи:
```sh
.venv/bin/python experiments/run.py prepare --reuse-images "$CAMPAIGN"
# Подставь новый путь и повтори run → summarize → plot.
```
Один `Ctrl+C` сохраняет данные и запускает уборку. Активные запросы завершаются в пределах таймаута. Продолжить только ещё не начатые прогоны:
```sh
.venv/bin/python experiments/run.py run "$CAMPAIGN" --resume
```
Прерванные попытки остаются в evidence и могут дать `inconclusive`; для полного повторения нужна новая серия. Не менять seal/state вручную.
`run ... --keep-volumes` оставляет экспериментальные volumes для отладки; при resume сохрани тот же флаг. После разбора, аварийного завершения WSL или ошибки уборки:
```sh
.venv/bin/python experiments/run.py cleanup "$CAMPAIGN"
```
Удаляются только ресурсы выбранной серии, evidence остаётся. При обычном завершении уборка автоматическая. Исходники и документы входят в manifest: их изменение требует нового prepare. Для summarize старой серии `analysis.py` и `common.py` должны совпадать с архивом `source.tar.gz`; старые образы сохраняй через `docker image save/load`.
+117
View File
@@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Plot saved summaries without changing raw evidence or the H1 decision."""
import argparse
import datetime
import hashlib
import json
import platform
import sys
from pathlib import Path
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import pandas as pd
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
from common import campaign_path
def sha256(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def create_plots(campaign):
inputs = [campaign / "metrics.csv", campaign / "metrics.json"]
frame = pd.read_csv(inputs[0])
summary = json.loads(inputs[1].read_text())
protocol = json.loads((campaign / "protocol.json").read_text())
profiles = [p["name"] for p in protocol["profiles"]]
status = summary["decision"]["criterion_status"]
output = campaign / "plots" / datetime.datetime.now(
datetime.timezone.utc
).strftime("%Y%m%dT%H%M%S.%fZ")
output.mkdir(parents=True)
def finish(fig, ax, title, ylabel, name):
ax.set_title(title + "\n" + status)
ax.set_ylabel(ylabel)
ax.set_xticks(range(len(profiles)), profiles, rotation=20, ha="right")
ax.grid(axis="y", alpha=0.25)
ax.legend(loc="best", fontsize=8)
fig.tight_layout()
fig.savefig(output / (name + ".png"), dpi=160)
plt.close(fig)
def points(column, scale, title, ylabel, name, limit=None):
fig, ax = plt.subplots(figsize=(10, 5))
for mode, offset, color in [("proxy", -0.12, "#2563eb"),
("direct", 0.12, "#d97706")]:
subset = frame[frame["mode"] == mode]
x, y = [], []
for index, profile in enumerate(profiles):
values = pd.to_numeric(
subset.loc[subset["profile"] == profile, column], errors="coerce"
).dropna().tolist()
for j, value in enumerate(values):
x.append(index + offset + (j - (len(values) - 1) / 2) * 0.035)
y.append(value / scale)
ax.scatter(x, y, label=mode, color=color, alpha=0.85)
if limit is not None:
ax.axhline(limit, color="red", linestyle="--", label="API limit")
finish(fig, ax, title + " (each point = one run)", ylabel, name)
points("api_bytes_per_ready", 1024 ** 2, "API RX+TX per ready attachment",
"MiB / ready", "api-traffic")
points("api_sampled_peak_window_max", 1024 ** 2,
"API sampled memory peak: maximum of two replicas", "MiB",
"api-memory-window", limit=512)
points("api_peak_lifetime_max", 1024 ** 2,
"API lifetime memory peak (includes startup and warmup)", "MiB",
"api-memory-lifetime", limit=512)
points("p95_seconds", 1, "Successful upload flow p95", "seconds", "latency-p95")
points("errors", 1, "Upload errors (missing runs are absent)", "count", "errors")
fig, ax = plt.subplots(figsize=(10, 5))
pairs = summary["decision"].get("pairs", [])
for valid, marker, label in [(True, "o", "instrumentation valid"),
(False, "x", "instrumentation issues")]:
selected = [p for p in pairs if p["instrumentation_valid"] == valid]
ax.scatter([profiles.index(p["profile"]) +
(p["repetition"] - 2) * 0.06 for p in selected],
[p["direct_proxy_ratio"] for p in selected],
marker=marker, label=label)
ax.axhline(protocol["criteria"]["median_direct_proxy_traffic_ratio_max"],
linestyle="--", color="green", label="median threshold")
ax.axhline(protocol["criteria"]["each_pair_traffic_ratio_max"],
linestyle=":", color="red", label="each pair threshold")
finish(fig, ax, "Paired API traffic ratio (all available pairs)",
"direct / proxy", "traffic-ratio")
provenance = {
"created_utc": datetime.datetime.now(datetime.timezone.utc).isoformat(),
"python": platform.python_version(), "pandas": pd.__version__,
"matplotlib": matplotlib.__version__,
"inputs_sha256": {f.name: sha256(f) for f in inputs + [campaign / "protocol.json"]},
"plot_script_sha256": sha256(Path(__file__)),
"requirements_sha256": sha256(Path(__file__).parent / "requirements.txt"),
"criterion_status": status,
"note": "All available runs shown; missing values omitted, never replaced with zero. "
"Instrumentation validity and system failures must be read in metrics.json.",
}
(output / "provenance.json").write_text(json.dumps(provenance, indent=2) + "\n")
return output
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("campaign", help="prepared campaign under root evidence/upload-comparison")
args = parser.parse_args()
try:
print(create_plots(campaign_path(args.campaign)))
except (OSError, ValueError, KeyError) as exc:
parser.exit(1, "Cannot plot: check campaign path and run summarize first (" +
type(exc).__name__ + ").\n")
if __name__ == "__main__":
main()
+15
View File
@@ -0,0 +1,15 @@
# Analysis environment (Python 3.12).
contourpy==1.4.0
cycler==0.12.1
fonttools==4.67.0
kiwisolver==1.5.1
matplotlib==3.9.2
numpy==2.5.4
packaging==26.3
pandas==2.2.3
pillow==13.0.0
pyparsing==3.3.3
python-dateutil==2.9.0.post0
pytz==2026.5
six==1.17.0
tzdata==2026.5
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env python3
"""WSL/Linux entry point. Nothing is measured until the explicit run command."""
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parent / "src"))
from driver import main
if __name__ == "__main__":
main()
+9
View File
@@ -0,0 +1,9 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
PYTHON="${PYTHON:-python3}"
"$PYTHON" -c 'import sys; sys.exit("Use Python 3.12 for the pinned analysis environment") if sys.version_info[:2] != (3, 12) else None'
"$PYTHON" -m venv "$ROOT/.venv"
"$ROOT/.venv/bin/python" -m pip install --no-compile -r "$ROOT/experiments/requirements.txt"
"$ROOT/.venv/bin/python" -m pip check
printf '\nReady: %s/.venv/bin/python\n' "$ROOT"
+392
View File
@@ -0,0 +1,392 @@
import csv
import json
import math
import statistics
from common import EXPERIMENT, ROOT, ExperimentError, digest, load, save
def percentile(values, fraction):
if not values:
return None
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)]
def json_lines(path):
if not path.exists():
return []
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
def case_metrics(directory, protocol):
case = load(directory / "case.json")
outcome = (
load(directory / "outcome.json")
if (directory / "outcome.json").exists()
else {"status": "interrupted"}
)
requests, verification = (
json_lines(directory / "requests.jsonl"),
json_lines(directory / "verification.jsonl"),
)
expected = protocol["clients"] * protocol["waves"]
result = {
**case,
"status": outcome["status"],
"expected_requests": expected,
"requests_recorded": len(requests),
"ready": sum(r.get("ready", False) for r in requests),
"errors": sum(
not r.get("ready", False) and r.get("error") != "interrupted"
for r in requests
),
"cancelled": sum(r.get("error") == "interrupted" for r in requests),
"verified": sum(v.get("verified", False) for v in verification),
"system_failures": [],
"instrumentation_issues": [],
"containers": [],
}
system, issues = result["system_failures"], result["instrumentation_issues"]
if result["errors"]:
system.append("upload_errors")
if (
len({r["attachment"]["id"] for r in requests if r.get("ready")})
!= result["ready"]
):
system.append("duplicate_attachment_ids")
if any(
not v.get("verified", False) and v.get("error") != "interrupted"
for v in verification
):
system.append("original_verification_failed")
warmups = json_lines(directory / "warmup.jsonl")
if any(
not r.get("ready", False) and r.get("error") != "interrupted" for r in warmups
):
system.append("warmup_upload_errors")
if len(requests) != expected or result["verified"] != result["ready"]:
issues.append("missing_request_or_verification_records")
if outcome["status"] != "completed":
issues.append("case_" + outcome["status"])
if outcome.get("sampler_errors"):
issues.append("sampler_error")
before = (
load(directory / "containers-before.json")
if (directory / "containers-before.json").exists()
else []
)
after = (
load(directory / "containers-after.json")
if (directory / "containers-after.json").exists()
else []
)
for c in after:
previous = next((p for p in before if p["id"] == c["id"]), None)
if (
c["oom_killed"]
or c["status"] != "running"
or (
previous
and (
c["restarts"] != previous["restarts"]
or c["started_at"] != previous["started_at"]
)
)
):
system.append("container_oom_restart_or_exit:" + c["service"])
if not (directory / "measurement.json").exists():
issues.append("missing_measurement")
return result
m = load(directory / "measurement.json")
seconds = (m["finished_ns"] - m["started_ns"]) / 1e9
result.update(
window_seconds=seconds,
generator_cpu_cores=m["generator_cpu_seconds"] / seconds,
ready_rps=result["ready"] / seconds,
)
durations = [
(r["finished_ns"] - r["started_ns"]) / 1e9 for r in requests if r.get("ready")
]
result["latency_seconds"] = {
"p50": percentile(durations, 0.50),
"p95": percentile(durations, 0.95),
"p99": percentile(durations, 0.99),
}
spreads = []
for wave in range(protocol["waves"]):
starts = [r["started_ns"] for r in requests if r["wave"] == wave]
if len(starts) == protocol["clients"]:
spreads.append((max(starts) - min(starts)) / 1e9)
result["max_start_spread_seconds"] = max(spreads, default=None)
criteria = protocol["criteria"]
if len(spreads) != protocol["waves"] or any(
s > criteria["start_spread_seconds_max"] for s in spreads
):
issues.append("burst_start_spread")
if result["generator_cpu_cores"] >= criteria["generator_cpu_cores_max"]:
issues.append("possible_generator_cpu_bottleneck")
instances = {r["attachment"]["instance"] for r in requests if r.get("ready")}
if len(instances) != protocol["resources"]["api_replicas"]:
issues.append("not_all_api_replicas_exercised")
warm_instances = {r["attachment"]["instance"] for r in warmups if r.get("ready")}
if len(warm_instances) != protocol["resources"]["api_replicas"]:
issues.append("not_all_api_replicas_warmed")
samples = json_lines(directory / "samples.jsonl")
api_traffic, background = 0, 0.0
for c in before:
cid = c["id"]
if cid not in m["before"] or cid not in m["after"]:
issues.append("missing_counter_snapshot:" + c["service"])
continue
start, end = m["before"][cid], m["after"][cid]
points = [
s
for s in samples
if s["container"] == cid
and m["started_ns"] <= s["monotonic_ns"] <= m["finished_ns"]
]
times = (
[m["started_ns"]] + [s["monotonic_ns"] for s in points] + [m["finished_ns"]]
)
gaps = [(b - a) / 1e9 for a, b in zip(times, times[1:])]
coverage = len(points) * protocol["sample_interval_seconds"] / seconds
rx, tx = (
end["rx_bytes"] - start["rx_bytes"],
end["tx_bytes"] - start["tx_bytes"],
)
if rx < 0 or tx < 0:
issues.append("network_counter_reset:" + c["service"])
idle_start, idle_end = m["idle_before"][cid], m["idle_after"][cid]
idle_seconds = (
idle_end["read_started_ns"] - idle_start["read_started_ns"]
) / 1e9
idle_rate = (
(idle_end["rx_bytes"] + idle_end["tx_bytes"])
- (idle_start["rx_bytes"] + idle_start["tx_bytes"])
) / idle_seconds
cpu_usec = end["cpu.stat"]["usage_usec"] - start["cpu.stat"]["usage_usec"]
peak = max(
[s["memory_current"] for s in points]
+ [start["memory.current"], end["memory.current"]]
)
events = {
key: end["memory.events"].get(key, 0) - start["memory.events"].get(key, 0)
for key in ("oom", "oom_kill", "max")
}
if end["memory.events"].get("oom", 0) or end["memory.events"].get(
"oom_kill", 0
):
system.append("oom_event:" + c["service"])
if (
coverage < criteria["sampler_coverage_fraction_min"]
or max(gaps) > criteria["sampler_gap_seconds_max"]
):
issues.append("sampler_coverage:" + c["service"])
item = {
"service": c["service"],
"hostname": c["hostname"],
"rx_bytes": rx,
"tx_bytes": tx,
"cpu_seconds": cpu_usec / 1e6,
"cpu_fraction_of_limit": cpu_usec / 1e6 / seconds / (c["nano_cpus"] / 1e9),
"throttled_seconds": (
end["cpu.stat"].get("throttled_usec", 0)
- start["cpu.stat"].get("throttled_usec", 0)
)
/ 1e6,
"throttled_periods": end["cpu.stat"].get("nr_throttled", 0)
- start["cpu.stat"].get("nr_throttled", 0),
"baseline_memory": start["memory.current"],
"sampled_peak_window": peak,
"peak_lifetime_before": start["memory.peak"],
"peak_lifetime_after": end["memory.peak"],
"memory_limit": end["memory.max"],
"memory_events_delta": events,
"idle_bytes_per_second": idle_rate,
"sample_count": len(points),
"sample_coverage": coverage,
"sample_gap_max_seconds": max(gaps),
}
result["containers"].append(item)
if c["service"] == "api":
api_traffic += rx + tx
background += idle_rate * seconds
if end["memory.peak"] >= criteria["api_peak_strictly_below_bytes"]:
system.append("api_lifetime_peak_reached_limit")
if sum(c["service"] == "api" for c in result["containers"]) != 2:
issues.append("missing_api_counters")
result["api_rx_tx_bytes"] = api_traffic
result["api_bytes_per_ready"] = (
api_traffic / result["ready"] if result["ready"] else None
)
api_metrics = [c for c in result["containers"] if c["service"] == "api"]
result["api_sampled_peak_window_max"] = max(
(c["sampled_peak_window"] for c in api_metrics), default=None
)
result["api_peak_lifetime_max"] = max(
(c["peak_lifetime_after"] for c in api_metrics), default=None
)
result["api_cpu_seconds_total"] = sum(c["cpu_seconds"] for c in api_metrics)
result["s3_rx_tx_bytes"] = sum(
c["rx_bytes"] + c["tx_bytes"]
for c in result["containers"]
if c["service"] == "s3"
)
result["estimated_background_fraction"] = background / (
expected * case["size_bytes"]
)
if (
result["estimated_background_fraction"]
> criteria["idle_background_payload_fraction_max"]
):
issues.append("excessive_background_traffic")
result["system_failures"] = sorted(set(system))
result["instrumentation_issues"] = sorted(set(issues))
return result
def decision(protocol, cases, plan):
pairs, issues, failures, profiles = [], [], [], []
if len(cases) != len(plan):
issues.append("not_all_planned_cases_finished")
for c in cases:
failures.extend(c["system_failures"])
issues.extend(c["instrumentation_issues"])
for profile in protocol["profiles"]:
valid_ratios = []
for repetition in range(1, protocol["repetitions"] + 1):
pair = [
c
for c in cases
if c["profile"] == profile["name"] and c["repetition"] == repetition
]
modes = {c["mode"]: c for c in pair}
if set(modes) != {"proxy", "direct"} or not all(
m.get("api_bytes_per_ready") for m in modes.values()
):
issues.append("incomplete_pair:" + profile["name"])
continue
ratio = (
modes["direct"]["api_bytes_per_ready"]
/ modes["proxy"]["api_bytes_per_ready"]
)
valid = all(not m["instrumentation_issues"] for m in modes.values())
if valid:
valid_ratios.append(ratio)
pairs.append(
{
"profile": profile["name"],
"repetition": repetition,
"direct_proxy_ratio": ratio,
"instrumentation_valid": valid,
}
)
profiles.append(
{
"profile": profile["name"],
"valid_pairs": len(valid_ratios),
"median_traffic_ratio": statistics.median(valid_ratios)
if valid_ratios
else None,
"min_traffic_ratio": min(valid_ratios, default=None),
"max_traffic_ratio": max(valid_ratios, default=None),
}
)
if any(
r > protocol["criteria"]["each_pair_traffic_ratio_max"]
for r in valid_ratios
) or (
len(valid_ratios) == protocol["repetitions"]
and statistics.median(valid_ratios)
> protocol["criteria"]["median_direct_proxy_traffic_ratio_max"]
):
failures.append("traffic_threshold:" + profile["name"])
if protocol["purpose"] == "pilot":
status = "pilot_not_for_hypothesis"
elif failures:
# OOM, errors and incorrect bytes are contrary outcomes, never exclusions.
status = "rejected"
elif issues:
status = "inconclusive"
else:
status = "confirmed_in_tested_conditions"
return {
"criterion_status": status,
"system_or_threshold_failures": sorted(set(failures)),
"instrumentation_issues": sorted(set(issues)),
"pairs": pairs,
"profiles": profiles,
}
def summarize(campaign):
manifest = load(campaign / "source-manifest.json")
analysis_sources = [EXPERIMENT / "src/analysis.py", EXPERIMENT / "src/common.py"]
if any(
manifest.get(str(path.relative_to(ROOT))) != digest(path)
for path in analysis_sources
):
raise ExperimentError(
"analysis code differs from frozen snapshot; use the archived experiment source"
)
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
cases = []
for c in plan:
directory = campaign / "cases" / c["directory"]
if (directory / "case.json").exists():
cases.append(case_metrics(directory, protocol))
result = {
"protocol_version": protocol["version"],
"purpose": protocol["purpose"],
"decision": decision(protocol, cases, plan),
"cases": cases,
}
if (campaign / "execution-failure.json").exists():
result["decision"]["instrumentation_issues"].append(
"campaign_execution_failure"
)
if result["decision"]["criterion_status"] == "confirmed_in_tested_conditions":
result["decision"]["criterion_status"] = "inconclusive"
save(campaign / "metrics.json", result)
save(
campaign / "analysis-provenance.json",
{
"source_sha256": {
str(p.relative_to(ROOT)): digest(p) for p in analysis_sources
},
"seal_sha256": digest(campaign / "seal.json"),
},
)
with (campaign / "metrics.csv").open("w", newline="") as output:
columns = [
"profile",
"mode",
"repetition",
"ready",
"errors",
"verified",
"api_bytes_per_ready",
"api_sampled_peak_window_max",
"api_peak_lifetime_max",
"api_cpu_seconds_total",
"s3_rx_tx_bytes",
"window_seconds",
"ready_rps",
"generator_cpu_cores",
"p50_seconds",
"p95_seconds",
"p99_seconds",
]
writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore")
writer.writeheader()
writer.writerows(
{
**case,
**{
name + "_seconds": value
for name, value in case.get("latency_seconds", {}).items()
},
}
for case in cases
)
return result
+273
View File
@@ -0,0 +1,273 @@
import base64
import hashlib
import http.client
import json
import secrets
import time
from urllib.parse import urlsplit
class FlowError(Exception):
def __init__(self, code):
self.code = code
class Client:
def __init__(self, base, protocol, stop):
self.base, self.protocol, self.stop = base, protocol, stop
def request(
self,
method,
url,
body=None,
token=None,
headers=None,
stage="api",
records=None,
deadline=None,
):
if self.stop.is_set():
raise FlowError("interrupted")
parsed = urlsplit(url)
if parsed.scheme != "http" or parsed.hostname not in ("localhost", "127.0.0.1"):
raise FlowError("unexpected_endpoint")
expected_port = self.protocol["ports"]["s3" if stage == "s3_post" else "api"]
if parsed.port != expected_port:
raise FlowError("unexpected_endpoint_port")
data = json.dumps(body).encode("utf-8") if isinstance(body, dict) else body
h = dict(headers or {})
if isinstance(body, dict):
h["Content-Type"] = "application/json"
if token:
h["Authorization"] = "Bearer " + token
remaining = (
deadline - time.monotonic()
if deadline
else self.protocol["request_timeout_seconds"]
)
if remaining <= 0:
raise FlowError("flow_timeout")
connection = http.client.HTTPConnection(
parsed.hostname,
parsed.port,
timeout=min(self.protocol["request_timeout_seconds"], remaining),
)
started = time.monotonic_ns()
record = {
"stage": stage,
"method": method,
"started_ns": started,
"request_body_bytes": len(data or b""),
}
if records is not None:
records.append(record)
try:
connection.request(
method,
parsed.path + (("?" + parsed.query) if parsed.query else ""),
data,
h,
)
response = connection.getresponse()
payload = response.read(65537)
record.update(
status=response.status,
response_body_bytes=len(payload),
instance=response.getheader("X-Instance-Id"),
request_id=response.getheader("X-Request-Id"),
)
if len(payload) > 65536:
raise FlowError("unexpected_response_size")
try:
decoded = json.loads(payload) if payload else None
except ValueError:
if response.status < 400:
raise FlowError("unexpected_response_format") from None
decoded = None
if response.status >= 400:
code = (
decoded.get("error", {}).get("code", "http_error")
if isinstance(decoded, dict)
else "http_" + str(response.status)
)
record["error"] = code
raise FlowError(code)
return decoded, response.status, record
except FlowError:
raise
except Exception as error:
record["error"] = type(error).__name__
raise FlowError(type(error).__name__) from None
finally:
record["finished_ns"] = time.monotonic_ns()
connection.close()
def api(self, method, path, **kwargs):
return self.request(method, self.base + path, **kwargs)
def bootstrap(self):
actors = []
for label in ("alice", "bob"):
username = "exp_" + label + "_" + secrets.token_hex(6)
password = secrets.token_hex(24)
user, status, _ = self.api(
"POST",
"/api/v1/users",
body={
"username": username,
"display_name": label,
"password": password,
},
)
if status != 201:
raise FlowError("registration_status")
basic = base64.b64encode((username + ":" + password).encode()).decode()
session, status, _ = self.api(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + basic},
)
if status != 201:
raise FlowError("session_status")
actors.append({"id": user["id"], "token": session["token"]})
chat, status, _ = self.api(
"POST",
"/api/v1/chats",
token=actors[0]["token"],
body={
"title": "Upload experiment",
"member_ids": [a["id"] for a in actors],
},
)
if status != 201:
raise FlowError("chat_status")
return actors, chat["id"]
def upload(self, mode, actor, chat, data, expected_digest, records, deadline):
path = "/api/v1/chats/" + chat
if mode == "proxy":
attachment, status, record = self.api(
"POST",
path + "/attachments?filename=experiment.png",
body=data,
token=actor["token"],
headers={"Content-Type": "image/png"},
stage="proxy",
records=records,
deadline=deadline,
)
else:
# Matches the browser: hash inside the measured logical direct flow.
start_hash = time.monotonic_ns()
checksum = hashlib.sha256(data).hexdigest()
records.append(
{
"stage": "client_sha256",
"started_ns": start_hash,
"finished_ns": time.monotonic_ns(),
}
)
instruction, status, _ = self.api(
"POST",
path + "/attachment-uploads",
token=actor["token"],
body={
"filename": "experiment.png",
"content_type": "image/png",
"size_bytes": len(data),
"sha256": checksum,
},
stage="init",
records=records,
deadline=deadline,
)
if status != 201:
raise FlowError("init_status")
fields = instruction["upload"]["fields"]
boundary = "experiment-" + secrets.token_hex(12)
pieces = [
f'--{boundary}\r\nContent-Disposition: form-data; name="{k}"\r\n\r\n{v}\r\n'.encode()
for k, v in fields.items()
]
pieces += [
f'--{boundary}\r\nContent-Disposition: form-data; name="file"; filename="experiment.png"\r\nContent-Type: image/png\r\n\r\n'.encode(),
data,
f"\r\n--{boundary}--\r\n".encode(),
]
multipart = b"".join(pieces)
# This is the only S3 request; no Bearer token or application headers.
_, post_status, _ = self.request(
"POST",
instruction["upload"]["url"],
body=multipart,
headers={"Content-Type": "multipart/form-data; boundary=" + boundary},
stage="s3_post",
records=records,
deadline=deadline,
)
if post_status != 204:
raise FlowError("s3_post_status")
attachment, status, record = self.api(
"POST",
path + "/attachment-uploads/" + instruction["upload_id"] + "/complete",
token=actor["token"],
stage="complete",
records=records,
deadline=deadline,
)
if (
status != 201
or attachment.get("status") != "ready"
or attachment.get("size_bytes") != len(data)
or attachment.get("sha256") != expected_digest
):
raise FlowError("incorrect_ready_metadata")
return {
"id": attachment["id"],
"sha256": attachment["sha256"],
"size_bytes": attachment["size_bytes"],
"instance": record["instance"],
}
def verify(self, actor, attachment, size, expected_digest):
if self.stop.is_set():
raise FlowError("interrupted")
parsed = urlsplit(self.base)
connection = http.client.HTTPConnection(
parsed.hostname, parsed.port, timeout=35
)
checksum, received = hashlib.sha256(), 0
try:
connection.request(
"GET",
"/api/v1/attachments/" + attachment["id"] + "/content",
headers={"Authorization": "Bearer " + actor["token"]},
)
response = connection.getresponse()
if (
response.status != 200
or response.getheader("Content-Length") != str(size)
or response.getheader("Content-Type") != "image/png"
):
raise FlowError("download_headers")
while True:
if self.stop.is_set():
raise FlowError("interrupted")
block = response.read(32768)
if not block:
break
received += len(block)
if received > size:
raise FlowError("download_size")
checksum.update(block)
if received != size or checksum.hexdigest() != expected_digest:
raise FlowError("download_checksum")
return {
"id": attachment["id"],
"verified": True,
"size_bytes": received,
"sha256": checksum.hexdigest(),
}
finally:
connection.close()
+112
View File
@@ -0,0 +1,112 @@
import hashlib
import json
import os
import subprocess
import tempfile
from pathlib import Path
EXPERIMENT = Path(__file__).resolve().parents[1]
ROOT = EXPERIMENT.parent
EVIDENCE = ROOT / "evidence" / "upload-comparison"
class ExperimentError(RuntimeError):
pass
def digest(path):
return hashlib.sha256(path.read_bytes()).hexdigest()
def save(path, value):
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(
json.dumps(value, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
os.replace(temporary, path)
def load(path):
return json.loads(path.read_text(encoding="utf-8"))
def command(args, log=None, timeout=300, input_text=None):
# Never format the exception with subprocess output: Docker can include
# environment details. Explicit build/start logs use commands without secrets.
result = subprocess.run(
args,
cwd=ROOT,
text=True,
input=input_text,
stdout=log or subprocess.PIPE,
stderr=subprocess.STDOUT,
timeout=timeout,
)
if result.returncode:
raise ExperimentError(
"command failed: " + args[0] + " (exit " + str(result.returncode) + ")"
)
return result.stdout.strip() if log is None else ""
def source_files():
fixed = [
ROOT / "src/Dockerfile",
ROOT / "src/.dockerignore",
ROOT / "src/go.mod",
ROOT / "src/go.sum",
ROOT / "infra/minio.Dockerfile",
ROOT / "infra/haproxy.cfg",
]
app = sorted(
p
for p in (ROOT / "src/backend").rglob("*")
if p.is_file() and p.suffix in (".go", ".sql")
)
harness = sorted(
p
for p in EXPERIMENT.rglob("*")
if p.is_file()
and "evidence" not in p.relative_to(EXPERIMENT).parts
and "__pycache__" not in p.parts
and p.suffix != ".pyc"
)
return sorted(set(fixed + app + harness))
def source_manifest():
return {str(p.relative_to(ROOT)): digest(p) for p in source_files()}
def check_sources(campaign):
expected = load(campaign / "source-manifest.json")
if source_manifest() != expected:
raise ExperimentError(
"source/config/docs changed since prepare; create a new campaign"
)
def check_seal(campaign):
seal = load(campaign / "seal.json")
for name, expected in seal.items():
if digest(campaign / name) != expected:
raise ExperimentError("frozen artifact changed: " + name)
for entry in load(campaign / "fixtures.json"):
if digest(campaign / "fixtures" / entry["filename"]) != entry["sha256"]:
raise ExperimentError("fixture checksum changed")
def campaign_path(value):
path = Path(value).expanduser().resolve()
if path.parent != EVIDENCE.resolve() or not (path / "protocol.json").is_file():
raise ExperimentError(
"campaign must be a prepared directory directly inside evidence/upload-comparison"
)
return path
def private_directory():
path = Path(tempfile.mkdtemp(prefix="highload-upload-exp-", dir="/tmp"))
path.chmod(0o700)
return path
+833
View File
@@ -0,0 +1,833 @@
import argparse
import concurrent.futures
import datetime
import fcntl
import hashlib
import json
import os
import platform
import resource
import secrets
import shutil
import signal
import socket
import subprocess
import sys
import tarfile
import threading
import time
import zlib
from pathlib import Path
from analysis import summarize
from client import Client, FlowError
from common import (
EVIDENCE,
EXPERIMENT,
ROOT,
ExperimentError,
campaign_path,
check_seal,
check_sources,
command,
digest,
load,
private_directory,
save,
source_manifest,
)
from fixtures import make_png
from telemetry import Sampler, inspect_containers, snapshot
def make_plan(protocol):
blocks = []
for index, profile in enumerate(protocol["profiles"]):
for repetition in range(1, protocol["repetitions"] + 1):
modes = (
["proxy", "direct"] if (index + repetition) % 2 else ["direct", "proxy"]
)
blocks.append((profile, repetition, modes))
blocks.sort(
key=lambda b: hashlib.sha256(
f"{protocol['seed']}:{b[0]['name']}:{b[1]}".encode()
).digest()
)
plan = []
for profile, repetition, modes in blocks:
for mode in modes:
number = len(plan) + 1
plan.append(
{
"number": number,
"profile": profile["name"],
"mode": mode,
"repetition": repetition,
"size_bytes": profile["size_bytes"],
"directory": f"{number:02d}-{profile['name']}-{mode}-r{repetition}",
}
)
return plan
def host_info():
info = json.loads(command(["docker", "info", "--format", "{{json .}}"], timeout=30))
if str(info.get("CgroupVersion")) != "2":
raise ExperimentError("Docker cgroup v2 is required")
safe = {
key: info.get(key)
for key in (
"ServerVersion",
"CgroupVersion",
"NCPU",
"MemTotal",
"KernelVersion",
"OperatingSystem",
"Driver",
)
}
return {
"python": sys.version,
"zlib": zlib.ZLIB_VERSION,
"platform": platform.platform(),
"docker": safe,
"compose": command(["docker", "compose", "version", "--short"]),
"client_cpu_affinity_count": len(os.sched_getaffinity(0))
if hasattr(os, "sched_getaffinity")
else os.cpu_count(),
"disk_free_bytes": shutil.disk_usage(EXPERIMENT).free,
}
def check_ports(protocol):
for port in protocol["ports"].values():
with socket.socket() as test:
try:
test.bind(("127.0.0.1", port))
except OSError:
raise ExperimentError(
"experiment port is occupied: " + str(port)
) from None
def preflight():
if platform.system() != "Linux":
raise ExperimentError("run inside WSL Ubuntu/Linux, not Windows Python")
protocol = load(EXPERIMENT / "config/protocol.json")
validate_compose(protocol)
info = host_info()
check_ports(protocol)
if not command(["docker", "compose", "version", "--short"]):
raise ExperimentError("Docker Compose plugin is required")
return info
def validate_compose(protocol):
if len(set(protocol["ports"].values())) != 3:
raise ExperimentError("API/S3/stats ports must differ")
values = {
"REPO_ROOT": str(ROOT),
"SECRET_DIR": "/tmp/experiment-config-check",
**{
role + "_IMAGE": "sha256:" + "0" * 64
for role in ("APP", "S3", "DB", "REDIS", "HAPROXY")
},
**{
role.upper() + "_PORT": str(port)
for role, port in protocol["ports"].items()
},
}
response = subprocess.run(
[
"docker",
"compose",
"--env-file",
"/dev/null",
"--project-name",
"hl-upload-exp-config-check",
"-f",
str(EXPERIMENT / "config/compose.yml"),
"config",
"--format",
"json",
],
check=False,
cwd=ROOT,
env={**os.environ, **values},
text=True,
capture_output=True,
timeout=30,
)
if response.returncode:
raise ExperimentError("experiment Compose config is invalid")
services = json.loads(response.stdout)["services"]
resources = protocol["resources"]
for role in ("api", "db", "s3", "redis", "haproxy"):
if (
float(services[role]["cpus"]) != resources[role + "_cpus"]
or int(services[role]["mem_limit"]) != resources[role + "_memory_bytes"]
):
raise ExperimentError("protocol/Compose resource mismatch: " + role)
if int(services[role]["memswap_limit"]) != int(services[role]["mem_limit"]):
raise ExperimentError("experiment swap must be disabled: " + role)
api = services["api"]
expected_env = {
"GOMAXPROCS": str(resources["api_gomaxprocs"]),
"GOMEMLIMIT": resources["api_go_memory_limit"],
"DATABASE_MAX_CONNS": str(resources["database_max_connections_per_replica"]),
}
if api["deploy"]["replicas"] != resources["api_replicas"] or any(
api["environment"].get(k) != v for k, v in expected_env.items()
):
raise ExperimentError("protocol/Compose API environment mismatch")
if (
api["tmpfs"] != ["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"]
or resources["api_tmpfs_bytes"] != 16777216
):
raise ExperimentError("unexpected tmpfs configuration")
def prepare(pilot=False, reuse=None):
info = preflight()
EVIDENCE.mkdir(parents=True, exist_ok=True)
stamp = datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
campaign = EVIDENCE / (
stamp + "-" + ("pilot" if pilot else "formal") + "-" + secrets.token_hex(3)
)
campaign.mkdir()
protocol = load(EXPERIMENT / "config/protocol.json")
if pilot:
protocol.update(
purpose="pilot", repetitions=1, clients=2, waves=1, warmup_clients=2
)
protocol["profiles"] = [p for p in protocol["profiles"] if p["width"] == 1024]
# Frozen before builds or any application requests.
save(campaign / "protocol.json", protocol)
save(campaign / "plan.json", make_plan(protocol))
sources = source_manifest()
save(campaign / "source-manifest.json", sources)
shutil.copyfile(EXPERIMENT / "config/compose.yml", campaign / "compose.yml")
with tarfile.open(campaign / "source.tar.gz", "w:gz") as archive:
for name in sources:
archive.add(ROOT / name, arcname=name, recursive=False)
save(campaign / "host-prepare.json", info)
save(
campaign / "git.json",
{
"head": command(["git", "rev-parse", "HEAD"]),
"branch": command(["git", "branch", "--show-current"]),
"status": command(["git", "status", "--short", "--untracked-files=no"]),
},
)
fixtures = []
for p in protocol["profiles"]:
filename = p["name"] + ".png"
data = make_png(p["width"], p["height"], p["size_bytes"], protocol["seed"])
(campaign / "fixtures").mkdir(exist_ok=True)
(campaign / "fixtures" / filename).write_bytes(data)
fixtures.append(
{**p, "filename": filename, "sha256": hashlib.sha256(data).hexdigest()}
)
save(campaign / "fixtures.json", fixtures)
try:
if reuse:
check_seal(reuse)
if load(reuse / "source-manifest.json") != sources:
raise ExperimentError(
"reuse-images source manifest differs; prepare without reuse"
)
images = load(reuse / "images.json")
for image in images.values():
command(["docker", "image", "inspect", image["id"]])
else:
refs = dict(protocol["images"])
app_hash = hashlib.sha256(
json.dumps(
{k: v for k, v in sources.items() if k.startswith("src/")},
sort_keys=True,
).encode()
).hexdigest()[:16]
refs["app"] = "highload-upload-experiment-app:" + app_hash
refs["s3"] = (
"highload-upload-experiment-s3:"
+ sources["infra/minio.Dockerfile"][:16]
)
with (campaign / "prepare.log").open("w") as log:
for role in ("db", "redis", "haproxy"):
try:
command(["docker", "image", "inspect", refs[role]])
except ExperimentError:
command(["docker", "pull", refs[role]], log, timeout=600)
command(
[
"docker",
"build",
"--tag",
refs["app"],
"--file",
str(ROOT / "src/Dockerfile"),
str(ROOT / "src"),
],
log,
timeout=1800,
)
command(
[
"docker",
"build",
"--tag",
refs["s3"],
"--file",
str(ROOT / "infra/minio.Dockerfile"),
str(ROOT / "infra"),
],
log,
timeout=1800,
)
images = {}
for role, ref in refs.items():
image = json.loads(command(["docker", "image", "inspect", ref]))[0]
images[role] = {
"id": image["Id"],
"reference": ref,
"repo_digests": image.get("RepoDigests", []),
"os": image["Os"],
"architecture": image["Architecture"],
"layers": image["RootFS"]["Layers"],
}
save(campaign / "images.json", images)
check_sources(campaign)
names = [
"protocol.json",
"plan.json",
"source-manifest.json",
"source.tar.gz",
"fixtures.json",
"images.json",
"compose.yml",
]
save(campaign / "seal.json", {name: digest(campaign / name) for name in names})
save(
campaign / "prepared.json",
{
"status": "prepared",
"purpose": protocol["purpose"],
"cases": len(load(campaign / "plan.json")),
},
)
except Exception as error:
save(
campaign / "prepare-failure.json",
{"error_type": type(error).__name__, "status": "not_prepared"},
)
raise
print(
"Prepared; no uploads measured: " + str(campaign.relative_to(ROOT)), flush=True
)
print("Seal: " + digest(campaign / "seal.json"), flush=True)
return campaign
def runtime_files(campaign):
directory = private_directory()
credentials = {
"db_password": secrets.token_hex(24),
"root_access": secrets.token_hex(12),
"root_secret": secrets.token_hex(32),
"api_access": secrets.token_hex(12),
"api_secret": secrets.token_hex(32),
}
credentials["database_url"] = (
"postgres://messenger_exp:"
+ credentials["db_password"]
+ "@db:5432/messenger_exp?sslmode=disable"
)
credentials["cache_url"] = "redis://redis:6379/0"
for name, value in credentials.items():
path = directory / name
path.write_text(value + "\n")
path.chmod(0o644) # private parent is 0700; bind individual files for app UID.
images, protocol = load(campaign / "images.json"), load(campaign / "protocol.json")
values = {
"REPO_ROOT": str(ROOT),
"SECRET_DIR": str(directory),
**{role.upper() + "_IMAGE": image["id"] for role, image in images.items()},
**{
role.upper() + "_PORT": str(port)
for role, port in protocol["ports"].items()
},
}
env = directory / "runtime.env"
env.write_text("".join(key + "=" + value + "\n" for key, value in values.items()))
env.chmod(0o600)
save(campaign / "runtime.json", {"private_directory": str(directory)})
return directory
def compose(campaign, runtime, project):
if not project.startswith("hl-upload-exp-") or len(project) > 63:
raise ExperimentError("unsafe experiment project name")
return [
"docker",
"compose",
"--project-name",
project,
"--env-file",
str(runtime / "runtime.env"),
"-f",
str(campaign / "compose.yml"),
]
def safe_down(campaign, runtime, project, log, keep_volumes):
# Every deletion is scoped by a unique project label, never a global prune.
allowed = {
project_name(campaign, case["number"]) for case in load(campaign / "plan.json")
}
if project not in allowed:
raise ExperimentError("refusing cleanup of another campaign's project")
for kind in ("container", "volume", "network"):
args = [
"docker",
kind,
"ls",
"--filter",
"label=com.docker.compose.project=" + project,
"-q",
]
if kind == "container":
args.insert(3, "-a")
ids = command(args).splitlines()
if ids:
items = json.loads(command(["docker", kind, "inspect", *ids]))
for item in items:
labels = (
item.get("Labels")
if kind != "container"
else item["Config"].get("Labels")
)
if (labels or {}).get("com.docker.compose.project") != project:
raise ExperimentError(
"refusing cleanup outside the selected project"
)
args = compose(campaign, runtime, project) + ["down", "--timeout", "15"]
if not keep_volumes:
args.append("--volumes")
command(args, log, timeout=120)
def project_name(campaign, number):
return (
"hl-upload-exp-"
+ campaign.name.lower().replace("t", "").replace("z", "")
+ "-"
+ str(number)
)
def host_activity():
result = {"loadavg": list(os.getloadavg()), "monotonic_ns": time.monotonic_ns()}
for row in Path("/proc/meminfo").read_text().splitlines():
if row.startswith(("MemAvailable:", "SwapFree:", "SwapTotal:")):
key, value = row.split(":", 1)
result[key + "_bytes"] = int(value.split()[0]) * 1024
return result
def batch(client, mode, actor, chat, data, digest_value, count, wave, output, stop):
barrier = threading.Barrier(count + 1)
def worker(index):
records = []
barrier.wait(timeout=15)
started = time.monotonic_ns()
result = {
"wave": wave,
"client": index,
"started_ns": started,
"ready": False,
"stages": records,
}
try:
attachment = client.upload(
mode,
actor,
chat,
data,
digest_value,
records,
time.monotonic() + client.protocol["flow_timeout_seconds"],
)
result.update(ready=True, attachment=attachment)
except FlowError as error:
result["error"] = error.code
except Exception as error:
result["error"] = type(error).__name__
result["finished_ns"] = time.monotonic_ns()
return result
results = []
with concurrent.futures.ThreadPoolExecutor(max_workers=count) as pool:
futures = [pool.submit(worker, index) for index in range(count)]
barrier.wait(timeout=15)
for future in concurrent.futures.as_completed(futures):
result = future.result()
output.write(json.dumps(result) + "\n")
output.flush()
results.append(result)
return results
def run_case(campaign, case, runtime, project, stop):
directory = campaign / "cases" / case["directory"]
directory.mkdir(parents=True)
save(directory / "case.json", {**case, "project": project})
protocol, images = load(campaign / "protocol.json"), load(campaign / "images.json")
cmd = compose(campaign, runtime, project)
sampler, containers, outcome = (
None,
[],
{"status": "setup_failed", "stage": "startup"},
)
try:
save(directory / "host-before.json", host_activity())
with (directory / "startup.log").open("w") as log:
command(
cmd + ["up", "-d", "--no-build", "--wait", "--wait-timeout", "120"],
log,
timeout=180,
)
ids = command(cmd + ["ps", "-q", "api", "s3"]).splitlines()
containers = inspect_containers(ids, project)
apis = [c for c in containers if c["service"] == "api"]
if len(apis) != 2 or len(containers) != 3:
raise ExperimentError("expected two API containers and one S3 container")
for c in containers:
expected = images["app" if c["service"] == "api" else "s3"]["id"]
if c["image"] != expected:
raise ExperimentError("unexpected image ID")
for c in apis:
if (
c["memory_limit"] != protocol["resources"]["api_memory_bytes"]
or c["memory_and_swap_limit"] != c["memory_limit"]
or c["nano_cpus"] != 500000000
or "size=16m" not in c["tmpfs"].get("/tmp/uploads", "")
):
raise ExperimentError("API resource configuration changed")
outcome["stage"] = "bootstrap_warmup"
client = Client(
"http://127.0.0.1:" + str(protocol["ports"]["api"]), protocol, stop
)
actors, chat = client.bootstrap()
fixture = next(
p for p in load(campaign / "fixtures.json") if p["name"] == case["profile"]
)
data = (campaign / "fixtures" / fixture["filename"]).read_bytes()
with (directory / "warmup.jsonl").open("w") as output:
warmup = batch(
client,
case["mode"],
actors[0],
chat,
data,
fixture["sha256"],
protocol["warmup_clients"],
-1,
output,
stop,
)
if not all(r["ready"] for r in warmup):
raise ExperimentError("warmup upload failed")
time.sleep(protocol["settle_seconds"])
sampler = Sampler(
containers, directory / "samples.jsonl", protocol["sample_interval_seconds"]
)
sampler.wait_ready()
idle_before = snapshot(containers)
time.sleep(protocol["idle_seconds"])
idle_after = snapshot(containers)
save(directory / "containers-before.json", inspect_containers(ids, project))
outcome["stage"] = "measured_uploads"
started = time.monotonic_ns()
cpu_before = resource.getrusage(resource.RUSAGE_SELF)
before = snapshot(containers)
requests = []
with (directory / "requests.jsonl").open("w") as output:
for wave in range(protocol["waves"]):
if stop.is_set():
break
requests.extend(
batch(
client,
case["mode"],
actors[0],
chat,
data,
fixture["sha256"],
protocol["clients"],
wave,
output,
stop,
)
)
if wave + 1 < protocol["waves"]:
stop.wait(protocol["wave_pause_seconds"])
after = snapshot(containers)
cpu_after = resource.getrusage(resource.RUSAGE_SELF)
finished = time.monotonic_ns()
save(
directory / "measurement.json",
{
"started_ns": started,
"finished_ns": finished,
"generator_cpu_seconds": cpu_after.ru_utime
+ cpu_after.ru_stime
- cpu_before.ru_utime
- cpu_before.ru_stime,
"generator_maxrss_kib": cpu_after.ru_maxrss,
"idle_before": idle_before,
"idle_after": idle_after,
"before": before,
"after": after,
},
)
sampler.close()
outcome["sampler_errors"] = sampler.errors
sampler = None
# Downloads must not contaminate measured API network counters.
outcome["stage"] = "verify_originals"
with (directory / "verification.jsonl").open("w") as output:
for request in requests:
if stop.is_set():
break
if not request["ready"]:
continue
try:
verified = client.verify(
actors[1], request["attachment"], len(data), fixture["sha256"]
)
except Exception as error:
verified = {
"id": request["attachment"]["id"],
"verified": False,
"error": error.code
if isinstance(error, FlowError)
else type(error).__name__,
}
output.write(json.dumps(verified) + "\n")
output.flush()
outcome.update(
status="interrupted" if stop.is_set() else "completed", stage="done"
)
except Exception as error:
outcome["error_type"] = type(error).__name__
if outcome["stage"] != "startup":
outcome["status"] = "instrumentation_or_setup_failure"
finally:
if sampler:
sampler.close()
outcome["sampler_errors"] = sampler.errors
if containers:
try:
save(
directory / "containers-after.json",
inspect_containers([c["id"] for c in containers], project),
)
except Exception as error:
outcome["inspect_error"] = type(error).__name__
save(directory / "host-after.json", host_activity())
save(directory / "outcome.json", outcome)
return outcome
def remove_runtime(runtime):
resolved = runtime.resolve()
if resolved.parent != Path("/tmp") or not resolved.name.startswith(
"highload-upload-exp-"
):
raise ExperimentError("refusing to remove unexpected private directory")
shutil.rmtree(resolved)
def run_campaign(campaign, resume=False, keep_volumes=False):
check_seal(campaign)
check_sources(campaign)
protocol, plan = load(campaign / "protocol.json"), load(campaign / "plan.json")
check_ports(protocol)
save(campaign / "host-run.json", host_info())
if (campaign / "cases").exists() and not resume:
raise ExperimentError(
"campaign already started; use --resume (does not overwrite attempts)"
)
if (campaign / "run-options.json").exists():
if load(campaign / "run-options.json")["keep_volumes"] != keep_volumes:
raise ExperimentError(
"resume must keep the original --keep-volumes setting; cleanup separately"
)
else:
save(campaign / "run-options.json", {"keep_volumes": keep_volumes})
if (campaign / "runtime.json").exists():
runtime = Path(load(campaign / "runtime.json")["private_directory"])
if not runtime.exists():
runtime = runtime_files(campaign)
else:
runtime = runtime_files(campaign)
stop = threading.Event()
previous = {}
for sig in (signal.SIGINT, signal.SIGTERM):
previous[sig] = signal.signal(sig, lambda *_: stop.set())
outcomes, cleanup_failed = [], False
invocation = (
datetime.datetime.now(datetime.UTC).strftime("%Y%m%dT%H%M%SZ")
+ "-"
+ secrets.token_hex(3)
)
try:
for case in plan:
if stop.is_set():
break
directory = campaign / "cases" / case["directory"]
if directory.exists():
print("Preserved existing attempt: " + case["directory"], flush=True)
continue
check_sources(campaign)
check_seal(campaign)
project = project_name(campaign, case["number"])
# Refuse reusing a project even after an earlier hard interruption.
if command(
[
"docker",
"volume",
"ls",
"-q",
"--filter",
"label=com.docker.compose.project=" + project,
]
):
raise ExperimentError(
"existing experiment volumes found; cleanup or prepare a new campaign"
)
print("Running " + case["directory"], flush=True)
try:
outcome = run_case(campaign, case, runtime, project, stop)
outcomes.append({"case": case["directory"], **outcome})
finally:
try:
with (directory / "shutdown.log").open("w") as log:
safe_down(campaign, runtime, project, log, keep_volumes)
except Exception as error:
cleanup_failed = True
save(
directory / "cleanup-failure.json",
{"error_type": type(error).__name__, "project": project},
)
raise ExperimentError(
"cleanup failed; inspect evidence and use cleanup command"
) from None
execution = {"interrupted": stop.is_set(), "last_invocation": outcomes}
save(campaign / "invocations" / (invocation + ".json"), execution)
save(campaign / "execution.json", execution)
check_sources(campaign)
print(
"Uploads finished; raw evidence retained. Run summarize; no human conclusion has been written.",
flush=True,
)
except Exception as error:
save(
campaign / "execution-failure.json",
{
"error_type": type(error).__name__,
"reason": str(error)
if isinstance(error, ExperimentError)
else "execution interrupted by instrumentation/environment failure",
},
)
raise
finally:
for sig, handler in previous.items():
signal.signal(sig, handler)
if not keep_volumes and not cleanup_failed:
remove_runtime(runtime)
else:
print(
"Private runtime retained outside evidence: " + str(runtime), flush=True
)
return 130 if stop.is_set() else 0
def cleanup_campaign(campaign):
runtime = Path(load(campaign / "runtime.json")["private_directory"])
if not runtime.exists():
raise ExperimentError(
"runtime already removed; normal run already removed its containers/volumes"
)
for case_file in sorted((campaign / "cases").glob("*/case.json")):
case = load(case_file)
with (case_file.parent / "manual-cleanup.log").open("a") as log:
safe_down(campaign, runtime, case["project"], log, False)
remove_runtime(runtime)
print("Only selected campaign projects removed; evidence kept.")
def main():
parser = argparse.ArgumentParser(
description="Reproducible proxy/presigned upload experiment (WSL/Linux, Python stdlib)"
)
sub = parser.add_subparsers(dest="action", required=True)
sub.add_parser("check", help="read-only prerequisites; no build or workload")
p = sub.add_parser(
"prepare", help="freeze protocol/source/fixtures and build images; no uploads"
)
p.add_argument("--pilot", action="store_true")
p.add_argument("--reuse-images", type=campaign_path)
for action in ("run", "summarize", "cleanup"):
p = sub.add_parser(action)
p.add_argument("campaign", type=campaign_path)
if action == "run":
p.add_argument("--resume", action="store_true")
p.add_argument("--keep-volumes", action="store_true")
args = parser.parse_args()
EVIDENCE.mkdir(parents=True, exist_ok=True)
with (EVIDENCE / ".lock").open("a") as lock:
try:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
raise SystemExit(
"Another experiment command is running; do not run campaigns concurrently."
)
try:
if args.action == "check":
info = preflight()
print(json.dumps(info, ensure_ascii=False, indent=2))
print(
"Prerequisites readable, experiment ports free; no uploads executed."
)
elif args.action == "prepare":
prepare(args.pilot, args.reuse_images)
elif args.action == "run":
raise SystemExit(
run_campaign(args.campaign, args.resume, args.keep_volumes)
)
elif args.action == "cleanup":
cleanup_campaign(args.campaign)
else:
check_seal(args.campaign)
result = summarize(args.campaign)
print(json.dumps(result["decision"], ensure_ascii=False, indent=2))
except (
ExperimentError,
OSError,
subprocess.TimeoutExpired,
ValueError,
KeyError,
) as error:
print(
"ERROR: "
+ (
str(error)
if isinstance(error, ExperimentError)
else type(error).__name__
),
file=sys.stderr,
)
raise SystemExit(1)
+35
View File
@@ -0,0 +1,35 @@
import hashlib
import struct
import zlib
def chunk(kind, payload):
return (
struct.pack(">I", len(payload))
+ kind
+ payload
+ struct.pack(">I", zlib.crc32(kind + payload) & 0xFFFFFFFF)
)
def make_png(width, height, size, seed):
compressor = zlib.compressobj(6)
compressed = []
for y in range(height):
color = bytes(((seed + y) % 256, (seed + y * 3) % 256, (seed + y * 7) % 256))
compressed.append(compressor.compress(b"\0" + color * width))
compressed.append(compressor.flush())
header = b"\x89PNG\r\n\x1a\n" + chunk(
b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
)
image = header + chunk(b"IDAT", b"".join(compressed))
# npAD: ancillary, private, reserved uppercase, unsafe-to-copy. The same
# decoded pixels occur at both encoded sizes; only this chunk grows.
remaining = size - len(image) - len(chunk(b"IEND", b"")) - 12
if remaining < 0:
raise ValueError("target size cannot hold this PNG")
block = hashlib.sha256(str(seed).encode("ascii")).digest()
padding = (block * ((remaining + len(block) - 1) // len(block)))[:remaining]
result = image + chunk(b"npAD", padding) + chunk(b"IEND", b"")
assert len(result) == size
return result
+171
View File
@@ -0,0 +1,171 @@
import json
import subprocess
import threading
import time
from common import ExperimentError, command
def parse_snapshot(text):
sections, current = {}, None
for line in text.splitlines():
if line.startswith("@"):
current = line[1:]
sections[current] = []
elif current:
sections[current].append(line)
metrics = {
name: int(sections[name][0])
for name in ("memory.current", "memory.peak", "memory.max")
}
metrics["memory.events"] = {
row.split()[0]: int(row.split()[1]) for row in sections["memory.events"]
}
metrics["cpu.stat"] = {
row.split()[0]: int(row.split()[1]) for row in sections["cpu.stat"]
}
interfaces = {}
for row in sections["net"]:
if ":" not in row:
continue
name, values = row.split(":", 1)
if name.strip() != "lo":
values = values.split()
interfaces[name.strip()] = {
"rx_bytes": int(values[0]),
"tx_bytes": int(values[8]),
}
if not interfaces:
raise ExperimentError("no non-loopback interfaces")
metrics["interfaces"] = interfaces
metrics["rx_bytes"] = sum(i["rx_bytes"] for i in interfaces.values())
metrics["tx_bytes"] = sum(i["tx_bytes"] for i in interfaces.values())
return metrics
def snapshot(containers):
result = {}
script = "for item in memory.current memory.peak memory.max memory.events cpu.stat; do printf '@%s\\n' \"$item\"; cat /sys/fs/cgroup/$item || exit 2; done; printf '@net\\n'; cat /proc/net/dev"
for item in containers:
start = time.monotonic_ns()
text = command(["docker", "exec", item["id"], "sh", "-c", script], timeout=15)
result[item["id"]] = {
**parse_snapshot(text),
"read_started_ns": start,
"read_finished_ns": time.monotonic_ns(),
}
return result
def inspect_containers(ids, project):
result = []
for item in json.loads(command(["docker", "inspect", *ids])):
labels = item["Config"]["Labels"]
if labels.get("com.docker.compose.project") != project:
raise ExperimentError("container belongs to a different project")
result.append(
{
"id": item["Id"],
"hostname": item["Config"]["Hostname"],
"service": labels["com.docker.compose.service"],
"image": item["Image"],
"started_at": item["State"]["StartedAt"],
"status": item["State"]["Status"],
"oom_killed": item["State"]["OOMKilled"],
"restarts": item["RestartCount"],
"memory_limit": item["HostConfig"]["Memory"],
"nano_cpus": item["HostConfig"]["NanoCpus"],
"memory_and_swap_limit": item["HostConfig"]["MemorySwap"],
"tmpfs": item["HostConfig"].get("Tmpfs", {}),
"volumes": [
m.get("Name") for m in item["Mounts"] if m["Type"] == "volume"
],
}
)
return result
class Sampler:
def __init__(self, containers, path, interval):
self.file = path.open("w", encoding="utf-8")
self.lock = threading.Lock()
self.items, self.threads, self.errors = [], [], []
self.counts = {c["id"]: 0 for c in containers}
self.stopping = False
# The in-container stop marker prevents docker-exec client termination
# from leaving a sampling shell running inside a retained container.
marker = "/tmp/upload-experiment-sampler.stop"
script = 'while [ ! -e "$1" ]; do read current < /sys/fs/cgroup/memory.current || exit 2; read peak < /sys/fs/cgroup/memory.peak || exit 2; read uptime rest < /proc/uptime; printf "%s %s %s\\n" "$current" "$peak" "$uptime"; sleep "$2"; done'
for c in containers:
process = subprocess.Popen(
[
"docker",
"exec",
c["id"],
"sh",
"-c",
script,
"sampler",
marker,
str(interval),
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.items.append((c, process, marker))
thread = threading.Thread(target=self.read, args=(c, process), daemon=True)
thread.start()
self.threads.append(thread)
def read(self, container, process):
try:
for line in process.stdout:
current, peak, uptime = line.split()
row = {
"container": container["id"],
"service": container["service"],
"monotonic_ns": time.monotonic_ns(),
"source_uptime_seconds": float(uptime),
"memory_current": int(current),
"memory_peak_lifetime": int(peak),
}
with self.lock:
self.counts[container["id"]] += 1
self.file.write(json.dumps(row) + "\n")
self.file.flush()
if not self.stopping:
self.errors.append("sampler exited before stop: " + container["id"])
except Exception as error:
self.errors.append(type(error).__name__)
def wait_ready(self):
deadline = time.monotonic() + 10
while not all(count >= 2 for count in self.counts.values()):
if self.errors or time.monotonic() > deadline:
raise ExperimentError("sampler did not become ready")
time.sleep(0.05)
def close(self):
self.stopping = True
for c, process, marker in self.items:
try:
command(["docker", "exec", c["id"], "touch", marker], timeout=3)
except Exception:
self.errors.append("cannot stop sampler: " + c["id"])
try:
process.wait(timeout=3)
except subprocess.TimeoutExpired:
process.terminate()
try:
process.wait(timeout=2)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
if process.returncode:
self.errors.append("sampler exit " + str(process.returncode))
for thread in self.threads:
thread.join(timeout=3)
if thread.is_alive():
self.errors.append("sampler reader did not stop")
self.file.close()
+416
View File
@@ -0,0 +1,416 @@
import copy
import hashlib
import json
import struct
import sys
import tempfile
import threading
import unittest
import zlib
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from common import EXPERIMENT, EVIDENCE, ROOT, save, load, ExperimentError
from driver import make_plan, safe_down, validate_compose
from fixtures import make_png
from telemetry import parse_snapshot
from analysis import percentile, decision, case_metrics
from client import Client
PROTOCOL = load(EXPERIMENT / "config/protocol.json")
class InstrumentTests(unittest.TestCase):
def test_evidence_is_in_repository_root(self):
self.assertEqual(EVIDENCE, ROOT / "evidence" / "upload-comparison")
def test_png_is_valid_deterministic_and_exact_size(self):
for width, height, size in [(16, 8, 4096), (32, 16, 8192)]:
data = make_png(width, height, size, 123)
self.assertEqual(data, make_png(width, height, size, 123))
self.assertEqual(len(data), size)
self.assertEqual(data[:8], b"\x89PNG\r\n\x1a\n")
chunks, offset = {}, 8
while offset < len(data):
length = struct.unpack(">I", data[offset : offset + 4])[0]
kind, payload = (
data[offset + 4 : offset + 8],
data[offset + 8 : offset + 8 + length],
)
crc = struct.unpack(
">I", data[offset + 8 + length : offset + 12 + length]
)[0]
self.assertEqual(crc, zlib.crc32(kind + payload) & 0xFFFFFFFF)
chunks[kind] = payload
offset += length + 12
self.assertEqual(offset, len(data))
self.assertEqual(struct.unpack(">II", chunks[b"IHDR"][:8]), (width, height))
self.assertEqual(
len(zlib.decompress(chunks[b"IDAT"])), height * (width * 3 + 1)
)
self.assertIn(b"npAD", chunks)
def test_all_actual_profile_headers_and_lengths(self):
for profile in PROTOCOL["profiles"]:
data = make_png(
profile["width"],
profile["height"],
profile["size_bytes"],
PROTOCOL["seed"],
)
self.assertEqual(len(data), profile["size_bytes"])
self.assertEqual(
struct.unpack(">II", data[16:24]), (profile["width"], profile["height"])
)
def test_plan_has_complete_counterbalanced_pairs(self):
plan = make_plan(PROTOCOL)
self.assertEqual(len(plan), 24)
self.assertEqual(plan, make_plan(PROTOCOL))
self.assertEqual(sum(p["mode"] == "proxy" for p in plan[::2]), 6)
self.assertEqual(len({p["directory"] for p in plan}), 24)
for profile in PROTOCOL["profiles"]:
for repetition in range(1, 4):
pair = [
c["mode"]
for c in plan
if c["profile"] == profile["name"] and c["repetition"] == repetition
]
self.assertEqual(sorted(pair), ["direct", "proxy"])
changed = copy.deepcopy(PROTOCOL)
changed["seed"] += 1
self.assertNotEqual(plan, make_plan(changed))
def test_counters_sum_all_non_loopback_interfaces(self):
text = """@memory.current
100
@memory.peak
200
@memory.max
536870912
@memory.events
oom 0
oom_kill 0
max 0
@cpu.stat
usage_usec 500
nr_throttled 1
throttled_usec 12
@net
Inter-| Receive | Transmit
lo: 900 0 0 0 0 0 0 0 800 0 0 0 0 0 0 0
eth0: 100 0 0 0 0 0 0 0 200 0 0 0 0 0 0 0
eth1: 300 0 0 0 0 0 0 0 400 0 0 0 0 0 0 0
"""
parsed = parse_snapshot(text)
self.assertEqual(parsed["rx_bytes"], 400)
self.assertEqual(parsed["tx_bytes"], 600)
self.assertEqual(parsed["cpu.stat"]["usage_usec"], 500)
def test_cleanup_cannot_target_another_campaign(self):
with tempfile.TemporaryDirectory() as folder:
campaign = Path(folder) / "series-formal-123"
campaign.mkdir()
save(campaign / "plan.json", make_plan(PROTOCOL))
with patch("driver.command") as command:
with self.assertRaises(ExperimentError):
safe_down(
campaign,
Path(folder),
"hl-upload-exp-another-series-1",
None,
False,
)
command.assert_not_called()
def test_compose_memory_values_may_be_strings(self):
services = {}
r = PROTOCOL["resources"]
for role in ("api", "db", "redis", "s3", "haproxy"):
services[role] = {
"cpus": r[role + "_cpus"],
"mem_limit": str(r[role + "_memory_bytes"]),
"memswap_limit": str(r[role + "_memory_bytes"]),
}
services["api"].update(
deploy={"replicas": 2},
tmpfs=["/tmp/uploads:rw,noexec,nosuid,size=16m,mode=1777"],
environment={
"GOMAXPROCS": "1",
"GOMEMLIMIT": "384MiB",
"DATABASE_MAX_CONNS": "10",
},
)
response = SimpleNamespace(
returncode=0, stdout=json.dumps({"services": services})
)
with patch("driver.subprocess.run", return_value=response):
validate_compose(PROTOCOL)
services["api"]["memswap_limit"] = "1073741824"
response.stdout = json.dumps({"services": services})
with patch("driver.subprocess.run", return_value=response):
with self.assertRaises(ExperimentError):
validate_compose(PROTOCOL)
class DecisionTests(unittest.TestCase):
def setUp(self):
self.plan = make_plan(PROTOCOL)
self.cases = [
{
**c,
"api_bytes_per_ready": 1100000 if c["mode"] == "direct" else 2000000,
"system_failures": [],
"instrumentation_issues": [],
}
for c in self.plan
]
def result(self, protocol=None):
return decision(protocol or PROTOCOL, self.cases, self.plan)
def test_declared_threshold_can_confirm(self):
self.assertEqual(
self.result()["criterion_status"], "confirmed_in_tested_conditions"
)
def test_bad_ratio_is_rejected(self):
for case in self.cases:
if case["mode"] == "direct":
case["api_bytes_per_ready"] = 1600000
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_one_bad_pair_cannot_hide_behind_median(self):
next(c for c in self.cases if c["mode"] == "direct")["api_bytes_per_ready"] = (
1420000
)
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_oom_is_not_discarded_when_sampler_also_failed(self):
self.cases[0]["system_failures"] = ["oom_event:api"]
self.cases[0]["instrumentation_issues"] = ["sampler_error"]
self.assertEqual(self.result()["criterion_status"], "rejected")
def test_broken_instruments_do_not_create_false_traffic_rejection(self):
direct = next(c for c in self.cases if c["mode"] == "direct")
direct["api_bytes_per_ready"] = 9000000
direct["instrumentation_issues"] = ["network_counter_reset:api"]
self.assertEqual(self.result()["criterion_status"], "inconclusive")
def test_missing_case_is_inconclusive(self):
self.cases.pop()
self.assertEqual(self.result()["criterion_status"], "inconclusive")
def test_pilot_never_confirms_the_hypothesis(self):
protocol = copy.deepcopy(PROTOCOL)
protocol["purpose"] = "pilot"
self.assertEqual(
self.result(protocol)["criterion_status"], "pilot_not_for_hypothesis"
)
def test_percentiles_use_nearest_rank(self):
self.assertEqual(percentile(list(range(1, 31)), 0.99), 30)
self.assertEqual(percentile(list(range(1, 31)), 0.50), 15)
self.assertIsNone(percentile([], 0.50))
def test_user_interruption_is_not_a_server_failure(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder)
save(path / "case.json", {**self.plan[0]})
save(path / "outcome.json", {"status": "interrupted"})
(path / "requests.jsonl").write_text(
json.dumps({"ready": False, "error": "interrupted"}) + "\n"
)
result = case_metrics(path, PROTOCOL)
self.assertEqual(result["errors"], 0)
self.assertEqual(result["cancelled"], 1)
self.assertEqual(result["system_failures"], [])
def test_metrics_keep_window_ram_distinct_and_exclude_s3_from_api_traffic(self):
protocol = copy.deepcopy(PROTOCOL)
protocol.update(clients=2, waves=1, sample_interval_seconds=0.1)
with tempfile.TemporaryDirectory() as folder:
path = Path(folder)
save(path / "case.json", self.plan[0])
save(path / "outcome.json", {"status": "completed"})
containers = [
{
"id": cid,
"hostname": cid,
"service": role,
"status": "running",
"started_at": "same",
"restarts": 0,
"oom_killed": False,
"nano_cpus": cpu,
}
for cid, role, cpu in [
("a", "api", 500000000),
("b", "api", 500000000),
("s", "s3", 1000000000),
]
]
save(path / "containers-before.json", containers)
save(path / "containers-after.json", containers)
def counters(received, sent, ns, peak):
return {
"rx_bytes": received,
"tx_bytes": sent,
"memory.current": 100,
"memory.peak": peak,
"memory.max": 536870912,
"memory.events": {"oom": 0, "oom_kill": 0, "max": 0},
"cpu.stat": {"usage_usec": ns // 10000},
"read_started_ns": ns,
"read_finished_ns": ns + 1,
}
ids = [c["id"] for c in containers]
m = {
"started_ns": 10000000000,
"finished_ns": 12000000000,
"generator_cpu_seconds": 0.1,
"idle_before": {cid: counters(0, 0, 0, 300) for cid in ids},
"idle_after": {cid: counters(0, 0, 5000000000, 300) for cid in ids},
"before": {cid: counters(0, 0, 10000000000, 300) for cid in ids},
"after": {
cid: counters(4000000, 4000000, 12000000000, 400) for cid in ids
},
}
save(path / "measurement.json", m)
requests = [
{
"wave": 0,
"ready": True,
"started_ns": 10100000000 + i * 1000000,
"finished_ns": 11900000000,
"attachment": {"id": cid, "instance": cid},
}
for i, cid in enumerate(("a", "b"))
]
(path / "requests.jsonl").write_text(
"".join(json.dumps(r) + "\n" for r in requests)
)
(path / "warmup.jsonl").write_text(
"".join(json.dumps(r) + "\n" for r in requests)
)
(path / "verification.jsonl").write_text(
"".join(
json.dumps({"id": cid, "verified": True}) + "\n"
for cid in ("a", "b")
)
)
(path / "samples.jsonl").write_text(
"".join(
json.dumps(
{
"container": cid,
"memory_current": 200,
"monotonic_ns": 10000000000 + i * 100000000,
}
)
+ "\n"
for cid in ids
for i in range(21)
)
)
result = case_metrics(path, protocol)
self.assertEqual(result["api_rx_tx_bytes"], 16000000)
self.assertEqual(result["api_bytes_per_ready"], 8000000)
self.assertEqual(result["containers"][0]["sampled_peak_window"], 200)
self.assertEqual(result["containers"][0]["peak_lifetime_after"], 400)
self.assertEqual(result["system_failures"], [])
self.assertEqual(result["instrumentation_issues"], [])
class TransportTests(unittest.TestCase):
def test_direct_stages_do_not_send_bearer_to_s3_or_leak_secrets_to_records(self):
observed, data = [], b"synthetic PNG transport fixture"
checksum = hashlib.sha256(data).hexdigest()
token, signed_field = "DO-NOT-PERSIST-TOKEN", "DO-NOT-PERSIST-SIGNATURE"
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_POST(self):
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
observed.append(
{
"path": self.path,
"authorization": self.headers.get("Authorization"),
"body": body,
}
)
if self.path.endswith("/attachment-uploads"):
response = {
"upload_id": "upload",
"upload": {
"url": f"http://127.0.0.1:{self.server.server_port}/bucket",
"fields": {
"key": "staging/direct/upload",
"policy": signed_field,
},
},
}
elif self.path == "/bucket":
self.send_response(204)
self.send_header("Content-Length", "0")
self.end_headers()
return
else:
response = {
"id": "attachment",
"status": "ready",
"size_bytes": len(data),
"sha256": checksum,
}
encoded = json.dumps(response).encode()
self.send_response(201)
self.send_header("Content-Length", str(len(encoded)))
self.send_header("X-Instance-Id", "api-test")
self.end_headers()
self.wfile.write(encoded)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
try:
protocol = copy.deepcopy(PROTOCOL)
protocol["ports"] = {"api": server.server_port, "s3": server.server_port}
client = Client(
f"http://127.0.0.1:{server.server_port}", protocol, threading.Event()
)
records = []
client.upload(
"direct",
{"token": token},
"chat",
data,
checksum,
records,
__import__("time").monotonic() + 10,
)
self.assertEqual(
[r["stage"] for r in records],
["client_sha256", "init", "s3_post", "complete"],
)
self.assertEqual(observed[0]["authorization"], "Bearer " + token)
self.assertIsNone(observed[1]["authorization"])
self.assertIn(data, observed[1]["body"])
self.assertEqual(observed[2]["authorization"], "Bearer " + token)
self.assertNotIn(token, json.dumps(records))
self.assertNotIn(signed_field, json.dumps(records))
finally:
server.shutdown()
server.server_close()
thread.join()
if __name__ == "__main__":
unittest.main()
+213
View File
@@ -0,0 +1,213 @@
import hashlib
import json
import sys
import tempfile
import threading
import unittest
from pathlib import Path
from unittest.mock import patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
import driver
from analysis import json_lines
from client import FlowError
from common import (
ROOT,
ExperimentError,
check_seal,
digest,
load,
save,
source_manifest,
)
class OrchestrationTests(unittest.TestCase):
def test_source_snapshot_excludes_secrets_evidence_and_git(self):
manifest = source_manifest()
self.assertIn("src/backend/services/direct_upload.go", manifest)
self.assertIn("experiments/docs/HYPOTHESIS.md", manifest)
for name in manifest:
self.assertNotIn(
name.split("/")[0], (".git", ".memory", "secrets", "evidence")
)
self.assertNotEqual(Path(name).name, ".env")
def test_prepare_seals_protocol_before_builds_and_does_not_upload(self):
with tempfile.TemporaryDirectory(dir=ROOT) as folder:
evidence = Path(folder)
names = [
"src/go.mod",
"infra/minio.Dockerfile",
"experiments/src/analysis.py",
"experiments/src/common.py",
]
manifest = {name: digest(ROOT / name) for name in names}
def fake_command(args, *_args, **_kwargs):
if args[:2] == ["docker", "build"]:
campaign = next(evidence.iterdir())
self.assertTrue((campaign / "protocol.json").exists())
self.assertFalse((campaign / "cases").exists())
return ""
if args[:3] == ["docker", "image", "inspect"]:
return json.dumps(
[
{
"Id": "sha256:" + "a" * 64,
"RepoDigests": [],
"Os": "linux",
"Architecture": "amd64",
"RootFS": {"Layers": ["layer"]},
}
]
)
if args[0] == "git":
return "synthetic-git-metadata"
raise AssertionError("unexpected command in prepare")
with (
patch("driver.EVIDENCE", evidence),
patch("driver.preflight", return_value={"purpose": "fake-host"}),
patch("driver.source_manifest", return_value=manifest),
patch("driver.check_sources"),
patch("driver.command", side_effect=fake_command),
patch("driver.Client") as client,
patch("builtins.print"),
):
campaign = driver.prepare(pilot=True)
client.assert_not_called()
check_seal(campaign)
self.assertEqual(load(campaign / "prepared.json")["cases"], 4)
self.assertEqual(load(campaign / "protocol.json")["purpose"], "pilot")
self.assertFalse((campaign / "cases").exists())
protocol = load(campaign / "protocol.json")
protocol["clients"] += 1
save(campaign / "protocol.json", protocol)
with self.assertRaises(ExperimentError):
check_seal(campaign)
def test_failed_uploads_are_saved_and_verification_starts_after_counters_close(
self,
):
with tempfile.TemporaryDirectory() as folder:
campaign = Path(folder)
protocol = load(driver.EXPERIMENT / "config/protocol.json")
protocol.update(
clients=2,
waves=1,
warmup_clients=2,
settle_seconds=0,
idle_seconds=0.001,
)
save(campaign / "protocol.json", protocol)
save(campaign / "images.json", {"app": {"id": "app"}, "s3": {"id": "s3"}})
data = b"synthetic-fixture"
checksum = hashlib.sha256(data).hexdigest()
save(
campaign / "fixtures.json",
[{"name": "fake", "filename": "fake.png", "sha256": checksum}],
)
(campaign / "fixtures").mkdir()
(campaign / "fixtures/fake.png").write_bytes(data)
case = {
"directory": "01-fake",
"number": 1,
"profile": "fake",
"mode": "direct",
"size_bytes": len(data),
"repetition": 1,
}
containers = [
{
"id": cid,
"hostname": cid,
"service": role,
"image": image,
"memory_limit": 536870912,
"memory_and_swap_limit": 536870912,
"nano_cpus": 500000000,
"tmpfs": {"/tmp/uploads": "rw,noexec,nosuid,size=16m,mode=1777"},
}
for cid, role, image in [
("a", "api", "app"),
("b", "api", "app"),
("s", "s3", "s3"),
]
]
state = {"snapshots": 0, "sampler_closed": False, "uploads": 0}
mutex = threading.Lock()
class FakeClient:
def __init__(self, _base, p, _stop):
self.protocol = p
def bootstrap(self):
return [{"token": "secret-a"}, {"token": "secret-b"}], "chat"
def upload(
self, _mode, _actor, _chat, _data, _digest, records, _deadline
):
with mutex:
state["uploads"] += 1
index = state["uploads"]
if index == 3:
raise FlowError("http_503")
return {
"id": "attachment-" + str(index),
"instance": "a" if index % 2 else "b",
}
def verify(self, _actor, attachment, _size, _digest):
if state["snapshots"] != 4 or not state["sampler_closed"]:
raise AssertionError("verification contaminated measurement")
return {"id": attachment["id"], "verified": True}
class FakeSampler:
errors = []
def __init__(self, _containers, path, _interval):
path.write_text("")
def wait_ready(self):
pass
def close(self):
state["sampler_closed"] = True
def fake_snapshot(_containers):
state["snapshots"] += 1
return {c["id"]: {"sample": "fake"} for c in containers}
def fake_command(args, *_args, **_kwargs):
return "a\nb\ns" if "ps" in args else ""
with (
patch("driver.command", side_effect=fake_command),
patch("driver.inspect_containers", return_value=containers),
patch("driver.Client", FakeClient),
patch("driver.Sampler", FakeSampler),
patch("driver.snapshot", side_effect=fake_snapshot),
):
outcome = driver.run_case(
campaign,
case,
Path(folder),
"hl-upload-exp-fake-1",
threading.Event(),
)
self.assertEqual(outcome["status"], "completed")
directory = campaign / "cases/01-fake"
requests = json_lines(directory / "requests.jsonl")
self.assertEqual(len(requests), 2)
self.assertEqual(sum(r["ready"] for r in requests), 1)
self.assertEqual(
next(r for r in requests if not r["ready"])["error"], "http_503"
)
self.assertEqual(len(json_lines(directory / "verification.jsonl")), 1)
self.assertNotIn("secret-a", (directory / "requests.jsonl").read_text())
self.assertTrue((directory / "measurement.json").exists())
if __name__ == "__main__":
unittest.main()
+49
View File
@@ -0,0 +1,49 @@
"""Offline plot checks; no application uploads or research measurements."""
import importlib.util
import json
import sys
import tempfile
import unittest
from pathlib import Path
AVAILABLE = all(importlib.util.find_spec(name) for name in ("pandas", "matplotlib"))
@unittest.skipUnless(AVAILABLE, "plot dependencies: run bash experiments/setup.sh")
class PlotTests(unittest.TestCase):
def test_plots_preserve_inputs_and_keep_all_attempts(self):
experiment = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("experiment_plot", experiment / "plot.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
with tempfile.TemporaryDirectory() as temporary:
campaign = Path(temporary)
protocol = {"profiles": [{"name": "sample"}], "criteria": {
"median_direct_proxy_traffic_ratio_max": 0.65,
"each_pair_traffic_ratio_max": 0.70}}
summary = {"decision": {"criterion_status": "inconclusive", "pairs": [
{"profile": "sample", "repetition": 1, "direct_proxy_ratio": 0.6,
"instrumentation_valid": False}]}}
(campaign / "protocol.json").write_text(json.dumps(protocol))
(campaign / "metrics.json").write_text(json.dumps(summary))
(campaign / "metrics.csv").write_text(
"profile,mode,repetition,errors,api_bytes_per_ready,"
"api_sampled_peak_window_max,api_peak_lifetime_max,p95_seconds\n"
"sample,proxy,1,1,2000000,200000000,300000000,2\n"
"sample,direct,1,0,1200000,210000000,310000000,3\n"
"sample,direct,2,2,,,,\n")
before = {f.name: f.read_bytes() for f in campaign.glob("*.json")}
before["metrics.csv"] = (campaign / "metrics.csv").read_bytes()
output = module.create_plots(campaign)
self.assertEqual(len(list(output.glob("*.png"))), 6)
self.assertEqual(len(list(output.glob("*.pdf"))), 0)
self.assertTrue(all(f.stat().st_size > 1000 for f in output.glob("*.png")))
provenance = json.loads((output / "provenance.json").read_text())
self.assertEqual(provenance["criterion_status"], "inconclusive")
for name, content in before.items():
self.assertEqual((campaign / name).read_bytes(), content)
self.assertNotEqual(output, module.create_plots(campaign))
if __name__ == "__main__":
unittest.main()