forked from mtusi2026/mtusi2026lab3
422 lines
17 KiB
Python
422 lines
17 KiB
Python
"""Open-loop HTTP experiment and concurrent one-second telemetry collection."""
|
|
|
|
import asyncio
|
|
import csv
|
|
import gzip
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
import random
|
|
import time
|
|
from collections import Counter
|
|
from datetime import datetime, timezone
|
|
|
|
import aiohttp
|
|
from common import (
|
|
BASE,
|
|
STATS,
|
|
current_containers,
|
|
inspect_public,
|
|
md5uuid,
|
|
sql,
|
|
write_json,
|
|
)
|
|
|
|
|
|
def percentile(values, quantile):
|
|
ordered = sorted(values)
|
|
return (
|
|
round(ordered[max(0, math.ceil(len(ordered) * quantile) - 1)], 3)
|
|
if ordered
|
|
else None
|
|
)
|
|
|
|
|
|
async def wait_ready(replicas):
|
|
async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=3)) as session:
|
|
for _ in range(100):
|
|
try:
|
|
async with session.get(STATS) as response:
|
|
rows = csv.DictReader(
|
|
io.StringIO((await response.text()).removeprefix("# "))
|
|
)
|
|
count = sum(
|
|
row["pxname"] == "api_replicas"
|
|
and row["svname"].startswith("api")
|
|
and row["status"].startswith("UP")
|
|
for row in rows
|
|
)
|
|
if count == replicas:
|
|
return
|
|
except (aiohttp.ClientError, TimeoutError):
|
|
pass
|
|
await asyncio.sleep(0.1)
|
|
raise RuntimeError("Not all expected replicas are ready in HAProxy")
|
|
|
|
|
|
async def load(actors, *, rate, duration, seed, prefix, limit=512):
|
|
rows, pending = [], set()
|
|
rng = random.Random(seed)
|
|
timeout = aiohttp.ClientTimeout(total=5, ceil_threshold=10)
|
|
connector = aiohttp.TCPConnector(limit=limit, limit_per_host=limit)
|
|
started = time.monotonic() + 0.05
|
|
cpu_started = time.process_time()
|
|
wall_started = time.time()
|
|
|
|
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
|
|
|
async def request(index, scheduled, chat_index, user_index):
|
|
begin = time.monotonic()
|
|
kind = "read" if index % 5 == 4 else "write"
|
|
chat_id = md5uuid("bench-chat-" + str(chat_index))
|
|
actor = actors[2 * chat_index + user_index]
|
|
key = hashlib.sha256(f"{prefix}:{index}".encode()).hexdigest()
|
|
text = key + "x" * 64
|
|
row = {
|
|
"index": index,
|
|
"kind": kind,
|
|
"scheduled_s": round(scheduled - started, 6),
|
|
"start_s": round(begin - started, 6),
|
|
"chat_id": chat_id,
|
|
"sender_id": actor["id"],
|
|
"key": key if kind == "write" else None,
|
|
}
|
|
try:
|
|
kwargs = {"headers": {"Authorization": "Bearer " + actor["token"]}}
|
|
method = "GET"
|
|
path = f"/api/v1/chats/{chat_id}/messages?limit=50"
|
|
if kind == "write":
|
|
method, path = "POST", path.split("?")[0]
|
|
kwargs["headers"]["Idempotency-Key"] = key
|
|
kwargs["json"] = {"text": text}
|
|
async with session.request(method, BASE + path, **kwargs) as response:
|
|
body = await response.read()
|
|
row.update(
|
|
status=response.status,
|
|
instance=response.headers.get("X-Instance-Id"),
|
|
)
|
|
expected = 201 if kind == "write" else 200
|
|
row["success"] = response.status == expected
|
|
if row["success"]:
|
|
payload = json.loads(body)
|
|
valid = bool(
|
|
row["instance"] and response.headers.get("X-Request-Id")
|
|
)
|
|
if kind == "write":
|
|
valid = (
|
|
valid
|
|
and payload.get("chat_id") == chat_id
|
|
and payload.get("sender_id") == actor["id"]
|
|
and payload.get("text") == text
|
|
)
|
|
row["message_id"] = payload.get("id")
|
|
valid = valid and bool(row["message_id"])
|
|
else:
|
|
valid = (
|
|
valid
|
|
and isinstance(payload.get("items"), list)
|
|
and len(payload["items"]) <= 50
|
|
)
|
|
if not valid:
|
|
row.update(success=False, error="invalid_response")
|
|
except (aiohttp.ClientError, TimeoutError, ValueError) as error:
|
|
row.update(success=False, error=type(error).__name__)
|
|
end = time.monotonic()
|
|
row.update(
|
|
end_s=round(end - started, 6),
|
|
latency_ms=1000 * (end - scheduled),
|
|
http_ms=1000 * (end - begin),
|
|
dispatch_lag_ms=1000 * (begin - scheduled),
|
|
)
|
|
rows.append(row)
|
|
|
|
for index in range(round(rate * duration)):
|
|
scheduled = started + index / rate
|
|
delay = scheduled - time.monotonic()
|
|
if delay > 0:
|
|
await asyncio.sleep(delay)
|
|
elif index % 50 == 0:
|
|
await asyncio.sleep(0)
|
|
chat_index, user_index = rng.randrange(100), rng.randrange(2)
|
|
if len(pending) >= limit:
|
|
rows.append(
|
|
{
|
|
"index": index,
|
|
"kind": "read" if index % 5 == 4 else "write",
|
|
"scheduled_s": index / rate,
|
|
"success": False,
|
|
"error": "generator_inflight_limit",
|
|
}
|
|
)
|
|
continue
|
|
task = asyncio.create_task(
|
|
request(index, scheduled, chat_index, user_index)
|
|
)
|
|
pending.add(task)
|
|
task.add_done_callback(pending.discard)
|
|
if pending:
|
|
await asyncio.gather(*pending)
|
|
elapsed = time.monotonic() - started
|
|
successes = [row for row in rows if row["success"]]
|
|
sent = [row for row in rows if "start_s" in row]
|
|
summary = {
|
|
"scheduled": len(rows),
|
|
"sent": len(sent),
|
|
"successes": len(successes),
|
|
"errors": len(rows) - len(successes),
|
|
"success_ratio": len(successes) / len(rows),
|
|
"requested_rps": rate,
|
|
"duration_s": duration,
|
|
"completion_elapsed_s": elapsed,
|
|
"goodput_rps": len(successes) / duration,
|
|
"window_goodput_rps": sum(row["end_s"] <= duration for row in successes)
|
|
/ duration,
|
|
"completion_goodput_rps": len(successes) / max(0.001, elapsed),
|
|
"achieved_send_rps": len(sent) / duration,
|
|
"p50_ms": percentile([row["latency_ms"] for row in successes], 0.5),
|
|
"p95_ms": percentile([row["latency_ms"] for row in successes], 0.95),
|
|
"p99_ms": percentile([row["latency_ms"] for row in successes], 0.99),
|
|
"dispatch_p95_ms": percentile([row["dispatch_lag_ms"] for row in sent], 0.95),
|
|
"http_p95_ms": percentile([row["http_ms"] for row in successes], 0.95),
|
|
"generator_dropped": len(rows) - len(sent),
|
|
"errors_by_type": dict(
|
|
Counter(
|
|
str(row.get("status", row.get("error")))
|
|
for row in rows
|
|
if not row["success"]
|
|
)
|
|
),
|
|
"instances": dict(Counter(row["instance"] for row in successes)),
|
|
"generator_cpu_cores": (time.process_time() - cpu_started)
|
|
/ max(0.001, elapsed),
|
|
"started_utc": datetime.fromtimestamp(wall_started, timezone.utc).isoformat(),
|
|
}
|
|
return rows, summary
|
|
|
|
|
|
DB_SAMPLE = """SELECT json_build_object(
|
|
'active', (SELECT count(*) FROM pg_stat_activity WHERE datname='benchmark' AND state='active' AND pid<>pg_backend_pid()),
|
|
'waits', (SELECT coalesce(json_agg(json_build_object('event_type',wait_event_type,'event',wait_event,'count',n)), '[]'::json)
|
|
FROM (SELECT wait_event_type,wait_event,count(*) n FROM pg_stat_activity WHERE datname='benchmark' AND pid<>pg_backend_pid() GROUP BY 1,2) t),
|
|
'database', (SELECT row_to_json(d) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,tup_inserted,deadlocks FROM pg_stat_database WHERE datname='benchmark') d)
|
|
);"""
|
|
|
|
|
|
async def sample_db(container):
|
|
proc = await asyncio.create_subprocess_exec(
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
container,
|
|
"psql",
|
|
"-X",
|
|
"-q",
|
|
"-t",
|
|
"-A",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-U",
|
|
"benchmark",
|
|
"-d",
|
|
"benchmark",
|
|
stdin=asyncio.subprocess.PIPE,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, _ = await proc.communicate(DB_SAMPLE.encode())
|
|
if proc.returncode:
|
|
raise RuntimeError("DB telemetry unavailable")
|
|
return json.loads(stdout)
|
|
|
|
|
|
async def collect(stop, containers, rows):
|
|
db = next(row["id"] for row in containers if row["service"] == "db")
|
|
connector = aiohttp.UnixConnector(path="/var/run/docker.sock")
|
|
async with (
|
|
aiohttp.ClientSession(
|
|
connector=connector, timeout=aiohttp.ClientTimeout(total=4)
|
|
) as engine,
|
|
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=4)) as http,
|
|
):
|
|
|
|
async def container_stats(container):
|
|
async with engine.get(
|
|
f"http://docker/v1.47/containers/{container['id']}/stats?stream=false&one-shot=true"
|
|
) as response:
|
|
data = await response.json()
|
|
return {
|
|
"service": container["service"],
|
|
"instance": container["instance"],
|
|
"cpu_stats": data.get("cpu_stats"),
|
|
"memory_stats": data.get("memory_stats"),
|
|
"blkio_stats": data.get("blkio_stats"),
|
|
}
|
|
|
|
while not stop.is_set():
|
|
begin = time.monotonic()
|
|
try:
|
|
resources, database = await asyncio.gather(
|
|
asyncio.gather(
|
|
*(container_stats(container) for container in containers)
|
|
),
|
|
sample_db(db),
|
|
)
|
|
haproxy, stats_error = [], None
|
|
try:
|
|
async with http.get(STATS) as response:
|
|
entries = csv.DictReader(
|
|
io.StringIO((await response.text()).removeprefix("# "))
|
|
)
|
|
haproxy = [
|
|
{
|
|
key: row.get(key)
|
|
for key in (
|
|
"pxname",
|
|
"svname",
|
|
"status",
|
|
"scur",
|
|
"qcur",
|
|
"qmax",
|
|
"econ",
|
|
"eresp",
|
|
"stot",
|
|
"addr",
|
|
"check_duration",
|
|
)
|
|
}
|
|
for row in entries
|
|
if row["pxname"] in ("api_replicas", "api_entry")
|
|
]
|
|
except (aiohttp.ClientError, TimeoutError) as error:
|
|
# Target stats may be unavailable under overload: that is a
|
|
# system observation, not grounds to discard poor throughput.
|
|
stats_error = type(error).__name__
|
|
# /proc/stat and process_time are raw inputs for CPU diagnostics.
|
|
with open("/proc/stat") as file:
|
|
host_cpu = file.readline().strip()
|
|
rows.append(
|
|
{
|
|
"time": datetime.now(timezone.utc).isoformat(),
|
|
"resources": resources,
|
|
"postgres": database,
|
|
"haproxy": haproxy,
|
|
"host_cpu": host_cpu,
|
|
"haproxy_stats_error": stats_error,
|
|
"generator_cpu_seconds": time.process_time(),
|
|
"collection_seconds": time.monotonic() - begin,
|
|
}
|
|
)
|
|
except Exception as error:
|
|
rows.append(
|
|
{
|
|
"time": datetime.now(timezone.utc).isoformat(),
|
|
"error": type(error).__name__,
|
|
}
|
|
)
|
|
try:
|
|
await asyncio.wait_for(
|
|
stop.wait(), timeout=max(0.01, 1 - (time.monotonic() - begin))
|
|
)
|
|
except TimeoutError:
|
|
pass
|
|
|
|
|
|
def verify_confirmed(rows):
|
|
successful = [row for row in rows if row["kind"] == "write" and row["success"]]
|
|
statement = "CREATE TEMP TABLE expected(id uuid,chat_id uuid,sender_id uuid,key text);\nCOPY expected FROM STDIN;\n"
|
|
statement += "".join(
|
|
"\t".join((row["message_id"], row["chat_id"], row["sender_id"], row["key"]))
|
|
+ "\n"
|
|
for row in successful
|
|
)
|
|
statement += "\\.\nSELECT json_build_object('expected',count(*),'matched',count(m.id),'linked',count(i.message_id)) FROM expected e LEFT JOIN messages m ON m.id=e.id AND m.chat_id=e.chat_id AND m.sender_id=e.sender_id AND m.text=e.key || repeat('x',64) LEFT JOIN message_idempotency i ON i.message_id=e.id AND i.sender_id=e.sender_id AND i.chat_id=e.chat_id AND i.key=e.key;"
|
|
check = json.loads(sql(statement))
|
|
check["passed"] = check["expected"] == check["matched"] == check["linked"]
|
|
return check
|
|
|
|
|
|
async def measure(
|
|
actors, directory, config, *, rate, duration, seed, prefix, warmup=15, limit=512
|
|
):
|
|
await wait_ready(config["replicas"])
|
|
warm_rows, warm_summary = await load(
|
|
actors,
|
|
rate=25,
|
|
duration=warmup,
|
|
seed=seed,
|
|
prefix=prefix + "-warm",
|
|
limit=limit,
|
|
)
|
|
write_json(directory / "warmup.json", warm_summary)
|
|
with gzip.open(
|
|
directory / "warmup-requests.jsonl.gz", "wt", encoding="utf-8"
|
|
) as file:
|
|
for row in warm_rows:
|
|
file.write(json.dumps(row, separators=(",", ":")) + "\n")
|
|
containers = current_containers()
|
|
stop, samples = asyncio.Event(), []
|
|
collection = asyncio.create_task(collect(stop, containers, samples))
|
|
started_utc = datetime.now(timezone.utc).isoformat()
|
|
try:
|
|
rows, summary = await load(
|
|
actors, rate=rate, duration=duration, seed=seed, prefix=prefix, limit=limit
|
|
)
|
|
finally:
|
|
stop.set()
|
|
await collection
|
|
for name, values in (("requests", rows), ("telemetry", samples)):
|
|
with gzip.open(
|
|
directory / (name + ".jsonl.gz"), "wt", encoding="utf-8"
|
|
) as file:
|
|
for row in values:
|
|
file.write(json.dumps(row, separators=(",", ":")) + "\n")
|
|
pools = []
|
|
for container in containers:
|
|
if container["service"] == "api":
|
|
# Go log output uses stderr; gather both without exposing other env.
|
|
import subprocess
|
|
|
|
result = subprocess.run(
|
|
["docker", "logs", "--since", started_utc, container["id"]],
|
|
check=False,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
for line in (result.stdout + result.stderr).splitlines():
|
|
if "benchmark_pool " in line:
|
|
value = json.loads(line.split("benchmark_pool ", 1)[1])
|
|
value["instance"] = container["instance"]
|
|
pools.append(value)
|
|
write_json(directory / "pool.json", pools)
|
|
summary["pool_max_by_instance"] = {row["instance"]: row["max"] for row in pools}
|
|
summary["gomaxprocs_by_instance"] = {
|
|
row["instance"]: row.get("gomaxprocs") for row in pools
|
|
}
|
|
summary["correctness"] = verify_confirmed(rows)
|
|
summary["telemetry_samples"] = len(samples)
|
|
summary["telemetry_errors"] = sum("error" in row for row in samples)
|
|
summary["haproxy_stats_errors"] = sum(
|
|
bool(row.get("haproxy_stats_error")) for row in samples
|
|
)
|
|
summary["containers_after"] = [
|
|
dict(inspect_public(container["id"]), service=container["service"])
|
|
for container in containers
|
|
]
|
|
cpu_limited = summary["generator_cpu_cores"] >= 0.85
|
|
summary["generator_limited"] = summary["generator_dropped"] / summary[
|
|
"scheduled"
|
|
] > 0.01 or (summary["dispatch_p95_ms"] > 100 and cpu_limited)
|
|
summary["valid"] = (
|
|
summary["telemetry_errors"] == 0 and not summary["generator_limited"]
|
|
)
|
|
summary["slo_pass"] = (
|
|
summary["valid"]
|
|
and summary["correctness"]["passed"]
|
|
and summary["success_ratio"] >= 0.99
|
|
and summary["p95_ms"] is not None
|
|
and summary["p95_ms"] <= 500
|
|
)
|
|
write_json(directory / "summary.json", summary)
|
|
return summary
|