323 lines
15 KiB
Python
323 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Reproducible, rate-controlled GET benchmark for the lab3 scaling track."""
|
|
|
|
import argparse
|
|
from collections import defaultdict
|
|
import http.client
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
from queue import Full, Queue
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
from urllib.request import urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402 - the repository's public bootstrap client
|
|
|
|
|
|
def topology(mode, replicas):
|
|
if mode not in ("fixed", "growth") or replicas not in (1, 2, 4):
|
|
raise ValueError("mode must be fixed/growth and replicas must be 1/2/4")
|
|
return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256)
|
|
|
|
|
|
def validate_matrix(rows):
|
|
if not rows:
|
|
raise ValueError("no benchmark rows")
|
|
rates = {row["rate"] for row in rows}
|
|
if len(rates) < 3:
|
|
raise ValueError("need at least three offered rates")
|
|
profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows}
|
|
if len(profiles) != 1:
|
|
raise ValueError("workload profile changed across measurements")
|
|
for mode in ("fixed", "growth"):
|
|
for replicas in (1, 2, 4):
|
|
_, cpu, memory = topology(mode, replicas)
|
|
for rate in rates:
|
|
repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)]
|
|
if sorted(repeats) != [1, 2, 3]:
|
|
raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}")
|
|
for row in rows:
|
|
if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory):
|
|
raise ValueError("incorrect API resource quota")
|
|
|
|
|
|
def percentile(values, fraction):
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3)
|
|
|
|
|
|
def command(*args):
|
|
return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip()
|
|
|
|
|
|
def compose(*args, override=None):
|
|
prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
|
|
if override:
|
|
prefix += ["-f", str(override)]
|
|
return command(*prefix, *args)
|
|
|
|
|
|
def pool_snapshot(base_url, replicas):
|
|
found = {}
|
|
for _ in range(8 * replicas):
|
|
with urlopen(base_url + "/health/ready", timeout=4) as response:
|
|
headers = response.headers
|
|
found[headers["X-Instance-Id"]] = {
|
|
"requests": int(headers["X-Pool-Requests"]),
|
|
"queued": int(headers["X-Pool-Queued"]),
|
|
"wait_ms": int(headers["X-Pool-Wait-Ms"]),
|
|
"size": int(headers["X-Pool-Size"]),
|
|
"available": int(headers["X-Pool-Available"]),
|
|
}
|
|
if len(found) == replicas:
|
|
break
|
|
return found
|
|
|
|
|
|
def db_snapshot(override):
|
|
sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback,"
|
|
"blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) "
|
|
"FROM pg_stat_database WHERE datname=current_database()")
|
|
values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override),
|
|
"exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
|
|
return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"),
|
|
map(int, values.split("|"))))
|
|
|
|
|
|
def docker_ids(override):
|
|
ids = {}
|
|
for service in ("api", "postgres", "lb"):
|
|
ids[service] = compose("ps", "-q", service, override=override).splitlines()
|
|
return ids
|
|
|
|
|
|
def stats_sample(ids):
|
|
all_ids = [id for service_ids in ids.values() for id in service_ids]
|
|
if not all_ids:
|
|
return {}
|
|
output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids)
|
|
by_id = {}
|
|
for line in output.splitlines():
|
|
item = json.loads(line)
|
|
by_id[item["ID"]] = item
|
|
result = {}
|
|
for service, service_ids in ids.items():
|
|
cpu = []
|
|
memory = []
|
|
for id in service_ids:
|
|
item = by_id.get(id[:12]) or by_id.get(id)
|
|
if not item:
|
|
raise ValueError("Docker stats omitted a benchmark container")
|
|
cpu.append(float(item["CPUPerc"].rstrip("%")))
|
|
amount = item["MemUsage"].split("/")[0].strip()
|
|
value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha())
|
|
memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024,
|
|
"kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3))
|
|
result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)}
|
|
return result
|
|
|
|
|
|
def open_loop(base_url, path, token, rate, duration, workers=64):
|
|
"""Schedule arrivals by clock; latency includes queue wait and HTTP time."""
|
|
address = urlsplit(base_url)
|
|
if address.scheme != "http":
|
|
raise ValueError("this local lab3 benchmark expects HTTP")
|
|
jobs = Queue(maxsize=256)
|
|
latencies = []
|
|
errors = defaultdict(int)
|
|
lock = threading.Lock()
|
|
starts = time.monotonic()
|
|
cpu_start = time.process_time()
|
|
count = math.ceil(rate * duration)
|
|
|
|
def worker():
|
|
connection = None
|
|
while True:
|
|
due = jobs.get()
|
|
if due is None:
|
|
jobs.task_done()
|
|
break
|
|
try:
|
|
if connection is None:
|
|
connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4)
|
|
connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"})
|
|
response = connection.getresponse()
|
|
payload = response.read()
|
|
if response.status != 200:
|
|
with lock:
|
|
errors[f"http_{response.status}"] += 1
|
|
elif len(json.loads(payload)["items"]) != 20:
|
|
with lock:
|
|
errors["wrong_page"] += 1
|
|
else:
|
|
with lock:
|
|
latencies.append((time.monotonic() - due) * 1000)
|
|
except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException):
|
|
with lock:
|
|
errors["transport_or_response"] += 1
|
|
if connection:
|
|
connection.close()
|
|
connection = None
|
|
finally:
|
|
jobs.task_done()
|
|
if connection:
|
|
connection.close()
|
|
|
|
threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)]
|
|
for thread in threads:
|
|
thread.start()
|
|
dropped = 0
|
|
for index in range(count):
|
|
due = starts + index / rate
|
|
delay = due - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
try:
|
|
jobs.put_nowait(due)
|
|
except Full:
|
|
dropped += 1
|
|
jobs.join()
|
|
for _ in threads:
|
|
jobs.put(None)
|
|
for thread in threads:
|
|
thread.join()
|
|
elapsed = time.monotonic() - starts
|
|
error_count = sum(errors.values()) + dropped
|
|
return {
|
|
"offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies),
|
|
"failed_requests": error_count, "error_ratio": round(error_count / count, 6),
|
|
"successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3),
|
|
"successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)},
|
|
"errors_by_type": dict(errors), "queue_dropped": dropped,
|
|
"client_cpu_seconds": round(time.process_time() - cpu_start, 3),
|
|
}
|
|
|
|
|
|
def fixture(base_url, size, fixture_file):
|
|
if fixture_file.exists():
|
|
saved = json.loads(fixture_file.read_text())
|
|
if saved["base_url"] != base_url or saved["dataset_messages"] != size:
|
|
raise ValueError("saved fixture URL or size differs from requested profile")
|
|
Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]})
|
|
return saved["token"], saved["path"]
|
|
client = Client(base_url)
|
|
alice, _, _, chat = client.bootstrap()
|
|
for index in range(size):
|
|
client.message(alice, chat, f"synthetic-scale-{index}")
|
|
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20"
|
|
fixture_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file:
|
|
file.write(json.dumps({"base_url": base_url, "dataset_messages": size,
|
|
"token": alice["token"], "path": path}) + "\n")
|
|
return alice["token"], path
|
|
|
|
|
|
def run(args):
|
|
full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4]
|
|
if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20:
|
|
raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages")
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
if args.output.exists():
|
|
raise ValueError(f"output already exists: {args.output}")
|
|
token = path = None
|
|
rows = []
|
|
with tempfile.TemporaryDirectory() as temporary:
|
|
override = Path(temporary) / "scale.yaml"
|
|
for mode in args.modes:
|
|
for replicas in args.replicas:
|
|
_, cpu, memory = topology(mode, replicas)
|
|
override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n")
|
|
compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override)
|
|
if token is None:
|
|
token, path = fixture(args.base_url, args.dataset_messages, args.fixture)
|
|
ids = docker_ids(override)
|
|
for rate in args.rates:
|
|
for repeat in range(1, args.repeats + 1):
|
|
open_loop(args.base_url, path, token, rate, args.warmup)
|
|
pool_before = pool_snapshot(args.base_url, replicas)
|
|
db_before = db_snapshot(override)
|
|
samples = []
|
|
sampling_errors = []
|
|
stop = threading.Event()
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
try:
|
|
samples.append(stats_sample(ids))
|
|
except (subprocess.CalledProcessError, ValueError, KeyError) as error:
|
|
sampling_errors.append(type(error).__name__)
|
|
return
|
|
stop.wait(1)
|
|
|
|
sampler = threading.Thread(target=sample, daemon=True)
|
|
sampler.start()
|
|
result = open_loop(args.base_url, path, token, rate, args.duration)
|
|
stop.set()
|
|
sampler.join()
|
|
if sampling_errors or not samples:
|
|
raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}")
|
|
db_after = db_snapshot(override)
|
|
if db_after["messages"] != db_before["messages"]:
|
|
raise RuntimeError("message dataset changed during read-only benchmark")
|
|
pool_after = pool_snapshot(args.base_url, replicas)
|
|
pool_delta = {
|
|
"observed_replicas": len(set(pool_before) & set(pool_after)),
|
|
"requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()),
|
|
"wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()),
|
|
"max_size": max((v["size"] for v in pool_after.values()), default=0),
|
|
}
|
|
services = {name: {
|
|
"avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3),
|
|
"max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3),
|
|
} for name in ("api", "postgres", "lb") if any(name in s for s in samples)}
|
|
row = {
|
|
"mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat,
|
|
"duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages,
|
|
"api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8,
|
|
**result, "docker": services, "pool": pool_delta,
|
|
"postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"},
|
|
"database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"],
|
|
}
|
|
rows.append(row)
|
|
with args.output.open("a") as output:
|
|
output.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True)
|
|
if full_matrix:
|
|
validate_matrix(rows)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500])
|
|
parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"])
|
|
parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4])
|
|
parser.add_argument("--duration", type=int, default=8)
|
|
parser.add_argument("--warmup", type=int, default=2)
|
|
parser.add_argument("--repeats", type=int, default=3)
|
|
parser.add_argument("--dataset-messages", type=int, default=1000)
|
|
parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json")
|
|
parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl")
|
|
args = parser.parse_args()
|
|
if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or
|
|
any(mode not in ("fixed", "growth") for mode in args.modes) or
|
|
any(replica not in (1, 2, 4) for replica in args.replicas) or
|
|
args.duration <= 0 or args.warmup <= 0):
|
|
raise ValueError("rates and durations must be positive")
|
|
run(args)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|