Files
mtusi2026lab3-fork/experiments/lab3-scale/load.py
T
2026-10-10 21:32:33 +03:00

422 lines
17 KiB
Python

"""Open-loop HTTP experiment and concurrent one-second telemetry collection."""
import asyncio
import csv
import gzip
import hashlib
import io
import json
import math
import random
import time
from collections import Counter
from datetime import datetime, timezone
import aiohttp
from common import (
BASE,
STATS,
current_containers,
inspect_public,
md5uuid,
sql,
write_json,
)
def percentile(values, quantile):
ordered = sorted(values)
return (
round(ordered[max(0, math.ceil(len(ordered) * quantile) - 1)], 3)
if ordered
else None
)
async def wait_ready(replicas):
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3)) as session:
for _ in range(100):
try:
async with session.get(STATS) as response:
rows = csv.DictReader(
io.StringIO((await response.text()).removeprefix("# "))
)
count = sum(
row["pxname"] == "api_replicas"
and row["svname"].startswith("api")
and row["status"].startswith("UP")
for row in rows
)
if count == replicas:
return
except (aiohttp.ClientError, TimeoutError):
pass
await asyncio.sleep(0.1)
raise RuntimeError("Not all expected replicas are ready in HAProxy")
async def load(actors, *, rate, duration, seed, prefix, limit=512):
rows, pending = [], set()
rng = random.Random(seed)
timeout = aiohttp.ClientTimeout(total=5, ceil_threshold=10)
connector = aiohttp.TCPConnector(limit=limit, limit_per_host=limit)
started = time.monotonic() + 0.05
cpu_started = time.process_time()
wall_started = time.time()
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
async def request(index, scheduled, chat_index, user_index):
begin = time.monotonic()
kind = "read" if index % 5 == 4 else "write"
chat_id = md5uuid("bench-chat-" + str(chat_index))
actor = actors[2 * chat_index + user_index]
key = hashlib.sha256(f"{prefix}:{index}".encode()).hexdigest()
text = key + "x" * 64
row = {
"index": index,
"kind": kind,
"scheduled_s": round(scheduled - started, 6),
"start_s": round(begin - started, 6),
"chat_id": chat_id,
"sender_id": actor["id"],
"key": key if kind == "write" else None,
}
try:
kwargs = {"headers": {"Authorization": "Bearer " + actor["token"]}}
method = "GET"
path = f"/api/v1/chats/{chat_id}/messages?limit=50"
if kind == "write":
method, path = "POST", path.split("?")[0]
kwargs["headers"]["Idempotency-Key"] = key
kwargs["json"] = {"text": text}
async with session.request(method, BASE + path, **kwargs) as response:
body = await response.read()
row.update(
status=response.status,
instance=response.headers.get("X-Instance-Id"),
)
expected = 201 if kind == "write" else 200
row["success"] = response.status == expected
if row["success"]:
payload = json.loads(body)
valid = bool(
row["instance"] and response.headers.get("X-Request-Id")
)
if kind == "write":
valid = (
valid
and payload.get("chat_id") == chat_id
and payload.get("sender_id") == actor["id"]
and payload.get("text") == text
)
row["message_id"] = payload.get("id")
valid = valid and bool(row["message_id"])
else:
valid = (
valid
and isinstance(payload.get("items"), list)
and len(payload["items"]) <= 50
)
if not valid:
row.update(success=False, error="invalid_response")
except (aiohttp.ClientError, TimeoutError, ValueError) as error:
row.update(success=False, error=type(error).__name__)
end = time.monotonic()
row.update(
end_s=round(end - started, 6),
latency_ms=1000 * (end - scheduled),
http_ms=1000 * (end - begin),
dispatch_lag_ms=1000 * (begin - scheduled),
)
rows.append(row)
for index in range(round(rate * duration)):
scheduled = started + index / rate
delay = scheduled - time.monotonic()
if delay > 0:
await asyncio.sleep(delay)
elif index % 50 == 0:
await asyncio.sleep(0)
chat_index, user_index = rng.randrange(100), rng.randrange(2)
if len(pending) >= limit:
rows.append(
{
"index": index,
"kind": "read" if index % 5 == 4 else "write",
"scheduled_s": index / rate,
"success": False,
"error": "generator_inflight_limit",
}
)
continue
task = asyncio.create_task(
request(index, scheduled, chat_index, user_index)
)
pending.add(task)
task.add_done_callback(pending.discard)
if pending:
await asyncio.gather(*pending)
elapsed = time.monotonic() - started
successes = [row for row in rows if row["success"]]
sent = [row for row in rows if "start_s" in row]
summary = {
"scheduled": len(rows),
"sent": len(sent),
"successes": len(successes),
"errors": len(rows) - len(successes),
"success_ratio": len(successes) / len(rows),
"requested_rps": rate,
"duration_s": duration,
"completion_elapsed_s": elapsed,
"goodput_rps": len(successes) / duration,
"window_goodput_rps": sum(row["end_s"] <= duration for row in successes)
/ duration,
"completion_goodput_rps": len(successes) / max(0.001, elapsed),
"achieved_send_rps": len(sent) / duration,
"p50_ms": percentile([row["latency_ms"] for row in successes], 0.5),
"p95_ms": percentile([row["latency_ms"] for row in successes], 0.95),
"p99_ms": percentile([row["latency_ms"] for row in successes], 0.99),
"dispatch_p95_ms": percentile([row["dispatch_lag_ms"] for row in sent], 0.95),
"http_p95_ms": percentile([row["http_ms"] for row in successes], 0.95),
"generator_dropped": len(rows) - len(sent),
"errors_by_type": dict(
Counter(
str(row.get("status", row.get("error")))
for row in rows
if not row["success"]
)
),
"instances": dict(Counter(row["instance"] for row in successes)),
"generator_cpu_cores": (time.process_time() - cpu_started)
/ max(0.001, elapsed),
"started_utc": datetime.fromtimestamp(wall_started, timezone.utc).isoformat(),
}
return rows, summary
DB_SAMPLE = """SELECT json_build_object(
'active', (SELECT count(*) FROM pg_stat_activity WHERE datname='benchmark' AND state='active' AND pid<>pg_backend_pid()),
'waits', (SELECT coalesce(json_agg(json_build_object('event_type',wait_event_type,'event',wait_event,'count',n)), '[]'::json)
FROM (SELECT wait_event_type,wait_event,count(*) n FROM pg_stat_activity WHERE datname='benchmark' AND pid<>pg_backend_pid() GROUP BY 1,2) t),
'database', (SELECT row_to_json(d) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,tup_inserted,deadlocks FROM pg_stat_database WHERE datname='benchmark') d)
);"""
async def sample_db(container):
proc = await asyncio.create_subprocess_exec(
"docker",
"exec",
"-i",
container,
"psql",
"-X",
"-q",
"-t",
"-A",
"-v",
"ON_ERROR_STOP=1",
"-U",
"benchmark",
"-d",
"benchmark",
stdin=asyncio.subprocess.PIPE,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, _ = await proc.communicate(DB_SAMPLE.encode())
if proc.returncode:
raise RuntimeError("DB telemetry unavailable")
return json.loads(stdout)
async def collect(stop, containers, rows):
db = next(row["id"] for row in containers if row["service"] == "db")
connector = aiohttp.UnixConnector(path="/var/run/docker.sock")
async with (
aiohttp.ClientSession(
connector=connector, timeout=aiohttp.ClientTimeout(total=4)
) as engine,
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=4)) as http,
):
async def container_stats(container):
async with engine.get(
f"http://docker/v1.47/containers/{container['id']}/stats?stream=false&one-shot=true"
) as response:
data = await response.json()
return {
"service": container["service"],
"instance": container["instance"],
"cpu_stats": data.get("cpu_stats"),
"memory_stats": data.get("memory_stats"),
"blkio_stats": data.get("blkio_stats"),
}
while not stop.is_set():
begin = time.monotonic()
try:
resources, database = await asyncio.gather(
asyncio.gather(
*(container_stats(container) for container in containers)
),
sample_db(db),
)
haproxy, stats_error = [], None
try:
async with http.get(STATS) as response:
entries = csv.DictReader(
io.StringIO((await response.text()).removeprefix("# "))
)
haproxy = [
{
key: row.get(key)
for key in (
"pxname",
"svname",
"status",
"scur",
"qcur",
"qmax",
"econ",
"eresp",
"stot",
"addr",
"check_duration",
)
}
for row in entries
if row["pxname"] in ("api_replicas", "api_entry")
]
except (aiohttp.ClientError, TimeoutError) as error:
# Target stats may be unavailable under overload: that is a
# system observation, not grounds to discard poor throughput.
stats_error = type(error).__name__
# /proc/stat and process_time are raw inputs for CPU diagnostics.
with open("/proc/stat") as file:
host_cpu = file.readline().strip()
rows.append(
{
"time": datetime.now(timezone.utc).isoformat(),
"resources": resources,
"postgres": database,
"haproxy": haproxy,
"host_cpu": host_cpu,
"haproxy_stats_error": stats_error,
"generator_cpu_seconds": time.process_time(),
"collection_seconds": time.monotonic() - begin,
}
)
except Exception as error:
rows.append(
{
"time": datetime.now(timezone.utc).isoformat(),
"error": type(error).__name__,
}
)
try:
await asyncio.wait_for(
stop.wait(), timeout=max(0.01, 1 - (time.monotonic() - begin))
)
except TimeoutError:
pass
def verify_confirmed(rows):
successful = [row for row in rows if row["kind"] == "write" and row["success"]]
statement = "CREATE TEMP TABLE expected(id uuid,chat_id uuid,sender_id uuid,key text);\nCOPY expected FROM STDIN;\n"
statement += "".join(
"\t".join((row["message_id"], row["chat_id"], row["sender_id"], row["key"]))
+ "\n"
for row in successful
)
statement += "\\.\nSELECT json_build_object('expected',count(*),'matched',count(m.id),'linked',count(i.message_id)) FROM expected e LEFT JOIN messages m ON m.id=e.id AND m.chat_id=e.chat_id AND m.sender_id=e.sender_id AND m.text=e.key || repeat('x',64) LEFT JOIN message_idempotency i ON i.message_id=e.id AND i.sender_id=e.sender_id AND i.chat_id=e.chat_id AND i.key=e.key;"
check = json.loads(sql(statement))
check["passed"] = check["expected"] == check["matched"] == check["linked"]
return check
async def measure(
actors, directory, config, *, rate, duration, seed, prefix, warmup=15, limit=512
):
await wait_ready(config["replicas"])
warm_rows, warm_summary = await load(
actors,
rate=25,
duration=warmup,
seed=seed,
prefix=prefix + "-warm",
limit=limit,
)
write_json(directory / "warmup.json", warm_summary)
with gzip.open(
directory / "warmup-requests.jsonl.gz", "wt", encoding="utf-8"
) as file:
for row in warm_rows:
file.write(json.dumps(row, separators=(",", ":")) + "\n")
containers = current_containers()
stop, samples = asyncio.Event(), []
collection = asyncio.create_task(collect(stop, containers, samples))
started_utc = datetime.now(timezone.utc).isoformat()
try:
rows, summary = await load(
actors, rate=rate, duration=duration, seed=seed, prefix=prefix, limit=limit
)
finally:
stop.set()
await collection
for name, values in (("requests", rows), ("telemetry", samples)):
with gzip.open(
directory / (name + ".jsonl.gz"), "wt", encoding="utf-8"
) as file:
for row in values:
file.write(json.dumps(row, separators=(",", ":")) + "\n")
pools = []
for container in containers:
if container["service"] == "api":
# Go log output uses stderr; gather both without exposing other env.
import subprocess
result = subprocess.run(
["docker", "logs", "--since", started_utc, container["id"]],
check=False,
text=True,
capture_output=True,
)
for line in (result.stdout + result.stderr).splitlines():
if "benchmark_pool " in line:
value = json.loads(line.split("benchmark_pool ", 1)[1])
value["instance"] = container["instance"]
pools.append(value)
write_json(directory / "pool.json", pools)
summary["pool_max_by_instance"] = {row["instance"]: row["max"] for row in pools}
summary["gomaxprocs_by_instance"] = {
row["instance"]: row.get("gomaxprocs") for row in pools
}
summary["correctness"] = verify_confirmed(rows)
summary["telemetry_samples"] = len(samples)
summary["telemetry_errors"] = sum("error" in row for row in samples)
summary["haproxy_stats_errors"] = sum(
bool(row.get("haproxy_stats_error")) for row in samples
)
summary["containers_after"] = [
dict(inspect_public(container["id"]), service=container["service"])
for container in containers
]
cpu_limited = summary["generator_cpu_cores"] >= 0.85
summary["generator_limited"] = summary["generator_dropped"] / summary[
"scheduled"
] > 0.01 or (summary["dispatch_p95_ms"] > 100 and cpu_limited)
summary["valid"] = (
summary["telemetry_errors"] == 0 and not summary["generator_limited"]
)
summary["slo_pass"] = (
summary["valid"]
and summary["correctness"]["passed"]
and summary["success_ratio"] >= 0.99
and summary["p95_ms"] is not None
and summary["p95_ms"] <= 500
)
write_json(directory / "summary.json", summary)
return summary