Files
mtusi2026lab1/student-tests/benchmark_memory.py
Самохвалов Андрей Ремович a6e90ad704
Application / application (push) Failing after 27s
Skeleton contracts / contracts (push) Failing after 41s
Write benchmark evidence with portable line endings
2026-10-02 17:11:53 +03:00

191 lines
7.9 KiB
Python

#!/usr/bin/env python3
"""Repeatable closed-loop read benchmark for the lab 1 in-memory history."""
import argparse
import base64
import csv
import http.client
import json
import math
import os
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
import subprocess
import threading
import time
from urllib.parse import quote
PORT = 18080
NAME = "mtusi-lab1-benchmark"
HISTORY = (100, 2000, 10000)
CLIENTS = (1, 4, 12)
FIELDS = (
"phase", "image", "history", "clients", "repeat", "seed", "duration_s", "warmup_s",
"cpu_limit", "memory_limit_mib", "success", "errors", "error_ratio", "rps",
"p50_ms", "p95_ms", "p99_ms", "cpu_percent_one_core", "rss_start_kib", "rss_end_kib",
)
def docker(*args):
return subprocess.check_output(("docker", *args), text=True).strip()
class HTTP:
def __init__(self):
self.conn = http.client.HTTPConnection("127.0.0.1", PORT, timeout=10)
def call(self, method, path, user=None, body=None):
headers = {"X-User-Id": user} if user else {}
if body is not None:
headers["Content-Type"] = "application/json"
body = json.dumps(body, separators=(",", ":")).encode()
self.conn.request(method, path, body, headers)
response = self.conn.getresponse()
data = response.read()
if response.status not in (200, 201):
raise RuntimeError(f"HTTP {response.status}: {data[:120]!r}")
return json.loads(data)
def close(self):
self.conn.close()
def seed_history(count, seed):
client = HTTP()
try:
user = client.call("POST", "/api/v1/users", body={"username": "bench", "display_name": "Benchmark"})["id"]
chat = client.call("POST", "/api/v1/chats", user, {"title": "Benchmark", "member_ids": [user]})["id"]
finally:
client.close()
def batch(worker):
local = HTTP()
try:
return [local.call("POST", f"/api/v1/chats/{chat}/messages", user,
{"text": f"synthetic-{seed}-{index:05d}"})
for index in range(worker, count, 8)]
finally:
local.close()
with ThreadPoolExecutor(max_workers=8) as pool:
messages = [message for batch_result in pool.map(batch, range(8)) for message in batch_result]
messages.sort(key=lambda item: (item["created_at"], item["id"]))
midpoint = messages[count // 2]
expected_ids = [message["id"] for message in messages[count // 2 + 1:count // 2 + 21]]
cursor = base64.urlsafe_b64encode(json.dumps(
[f"messages:{chat}", midpoint["created_at"], midpoint["id"]], separators=(",", ":")
).encode()).decode().rstrip("=")
path = f"/api/v1/chats/{chat}/messages?limit=20&cursor={quote(cursor)}"
client = HTTP()
try:
page = client.call("GET", path, user)
assert [message["id"] for message in page["items"]] == expected_ids
assert (page["items"][0]["created_at"], page["items"][0]["id"]) > (midpoint["created_at"], midpoint["id"])
finally:
client.close()
return user, path, expected_ids
def process_metrics():
stat = docker("exec", NAME, "cat", "/proc/1/stat").split()
status = docker("exec", NAME, "cat", "/proc/1/status")
rss = int(next(line.split()[1] for line in status.splitlines() if line.startswith("VmRSS:")))
return (int(stat[13]) + int(stat[14])) / os.sysconf("SC_CLK_TCK"), rss
def percentile(values, fraction):
return round(values[math.ceil(len(values) * fraction) - 1], 3) if values else ""
def load(user, path, expected_ids, clients, duration):
barrier = threading.Barrier(clients)
deadline = time.monotonic() + duration
def worker():
connection = HTTP()
latencies, errors = [], 0
barrier.wait()
try:
while time.monotonic() < deadline:
start = time.perf_counter()
try:
page = connection.call("GET", path, user)
if [message["id"] for message in page["items"]] != expected_ids:
raise ValueError("history page differs from seeded messages")
latencies.append((time.perf_counter() - start) * 1000)
except (OSError, RuntimeError, ValueError, KeyError, TypeError):
errors += 1
connection.close()
connection = HTTP()
finally:
connection.close()
return latencies, errors
start = time.monotonic()
with ThreadPoolExecutor(max_workers=clients) as pool:
results = list(pool.map(lambda _: worker(), range(clients)))
elapsed = time.monotonic() - start
latencies = sorted(value for successes, _ in results for value in successes)
errors = sum(failures for _, failures in results)
return elapsed, latencies, errors
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--phase", choices=("baseline", "after"), required=True)
parser.add_argument("--image", required=True)
parser.add_argument("--output", default="evidence/load-lab1.csv")
parser.add_argument("--duration", type=float, default=3)
parser.add_argument("--warmup", type=float, default=1)
parser.add_argument("--repeats", type=int, default=2)
parser.add_argument("--seed", type=int, default=20261001)
args = parser.parse_args()
if args.duration <= 0 or args.warmup <= 0 or args.repeats < 2:
parser.error("duration and warmup must be positive; repeats must be >= 2")
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
new_file = not output.exists()
with output.open("a", newline="") as result:
writer = csv.DictWriter(result, fieldnames=FIELDS, lineterminator="\n")
if new_file:
writer.writeheader()
for history in HISTORY:
docker("run", "-d", "--rm", "--name", NAME, "--cpus=1", "--memory=512m",
"--memory-swap=512m", "-p", f"127.0.0.1:{PORT}:8080", args.image)
try:
for _ in range(100):
try:
HTTP().call("GET", "/health/ready")
break
except OSError:
time.sleep(.1)
else:
raise RuntimeError("container did not become ready")
user, path, expected_ids = seed_history(history, args.seed)
for clients in CLIENTS:
for repeat in range(1, args.repeats + 1):
load(user, path, expected_ids, clients, args.warmup)
cpu_before, rss_before = process_metrics()
elapsed, latencies, errors = load(user, path, expected_ids, clients, args.duration)
cpu_after, rss_after = process_metrics()
total = len(latencies) + errors
row = dict(phase=args.phase, image=args.image, history=history, clients=clients,
repeat=repeat, seed=args.seed, duration_s=args.duration, warmup_s=args.warmup,
cpu_limit=1, memory_limit_mib=512, success=len(latencies), errors=errors,
error_ratio=round(errors / total, 6) if total else 0,
rps=round(len(latencies) / elapsed, 2),
p50_ms=percentile(latencies, .5), p95_ms=percentile(latencies, .95),
p99_ms=percentile(latencies, .99),
cpu_percent_one_core=round((cpu_after - cpu_before) / elapsed * 100, 1),
rss_start_kib=rss_before, rss_end_kib=rss_after)
writer.writerow(row)
result.flush()
print(row, flush=True)
finally:
docker("stop", "--time", "10", NAME)
if __name__ == "__main__":
main()