Files
mtusi2026lab3/research/run.py
T

415 lines
14 KiB
Python

#!/usr/bin/env python3
"""Repeatable scaling matrix on an isolated Compose project and fresh data."""
import argparse
import hashlib
import json
import os
import platform
import random
import subprocess
import sys
import tempfile
import threading
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import psycopg
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from messenger.cursor import encode_cursor
from messenger.repository import stamp
ROOT = Path(__file__).resolve().parents[1]
PROJECT = "mtusi-lab3-research"
K6 = "grafana/k6@sha256:3ddc8b1a33a2c3d8edc6e99b6a762ae36cba08788463458f5e6a7703e14eb77d"
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def dc(*args, env):
return subprocess.run(
[
"docker",
"compose",
"-p",
PROJECT,
"--env-file",
".env",
"-f",
"compose.yaml",
"-f",
"compose.test.yaml",
*args,
],
cwd=ROOT,
env=env,
check=True,
capture_output=True,
text=True,
timeout=120,
)
def count(metrics, key):
return metrics.get(key, {}).get("values", {}).get("count", 0)
def wait_api(client, expected):
seen, deadline = set(), time.monotonic() + 60
while time.monotonic() < deadline:
try:
response = client.call("GET", "/health/ready", schema="Health")
seen.add(response["headers"]["X-Instance-Id"])
if set(expected) <= seen:
return sorted(expected)
except Failure:
pass
time.sleep(0.1)
raise RuntimeError("Research APIs failed to become ready")
def samples(stop, container_ids, result):
while not stop.is_set():
try:
r = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *container_ids],
capture_output=True,
text=True,
check=True,
timeout=5,
)
result.append(
{
"at": datetime.now(UTC).isoformat(),
"containers": [
{k: value[k] for k in ("ID", "Name", "CPUPerc", "MemUsage", "MemPerc")}
for value in map(json.loads, r.stdout.splitlines())
],
}
)
except OSError, ValueError, KeyError, subprocess.SubprocessError:
result.append({"error": "resource_sample_unavailable"})
stop.wait(0.25)
def fixture(conn, client):
# This connection only belongs to PROJECT's separate PostgreSQL container.
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions,message_requests CASCADE")
alice, _, _, chat = client.bootstrap()
conn.execute(
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
"SELECT md5(%s || g::text)::uuid,%s,%s,repeat('m',128),"
"'2026-10-17T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
"FROM generate_series(1,10000) g",
(chat["id"], chat["id"], alice["id"]),
)
conn.execute("VACUUM ANALYZE messages")
row = conn.execute(
"SELECT created_at,id FROM messages WHERE chat_id=%s "
"ORDER BY created_at,id OFFSET 9000 LIMIT 1",
(chat["id"],),
).fetchone()
cursor = encode_cursor("messages:" + chat["id"], (stamp(row["created_at"]), str(row["id"])))
return {"token": alice["token"], "chat": chat["id"], "cursor": cursor, "run": uuid4().hex}
def run_trial(config, env, output, dsn, duration, warmup):
env = {
**env,
"API_REPLICAS": str(config["replicas"]),
"POSTGRES_CPU": str(config.get("postgres_cpu", 1)),
"API_CPU": str(config["cpu_each"]),
"API_MEMORY": str(config["memory_mib_each"]) + "m",
"DATABASE_READ_AUTOCOMMIT": str(int(config["optimized"])),
}
dc("up", "-d", "--no-build", "--wait", env=env)
# PostgreSQL CPU changes recreate its container; never reuse that old socket.
with psycopg.connect(dsn, autocommit=True, row_factory=dict_row) as conn:
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
measure(config, env, output, conn, duration, warmup)
def measure(config, env, output, conn, duration, warmup):
client = Client("http://127.0.0.1:18083", timeout=5)
ids = dc("ps", "-q", "api", env=env).stdout.split()
assert len(ids) == config["replicas"]
aliases = []
for container in ids:
code = (
"import urllib.request; r=urllib.request.urlopen("
"'http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"
)
aliases.append(
subprocess.check_output(
["docker", "exec", container, "python", "-c", code], text=True
).strip()
)
aliases = wait_api(client, aliases)
dc("exec", "-T", "valkey", "valkey-cli", "FLUSHDB", env=env)
data = fixture(conn, client)
data["instances"] = aliases
conn.execute("SELECT pg_stat_statements_reset()")
resources, stop, generator = [], threading.Event(), None
with tempfile.TemporaryDirectory(prefix="lab3-load-") as temporary:
temporary = Path(temporary)
session = temporary / "session.json"
session.write_text(json.dumps(data))
os.chmod(session, 0o600)
logs = temporary / "out"
logs.mkdir()
try:
r = subprocess.run(
[
"docker",
"run",
"-d",
"--network",
PROJECT + "_backend",
"--cpus",
"2",
"--memory",
"384m",
"--memory-swap",
"384m",
"--user",
str(os.getuid()) + ":" + str(os.getgid()),
"--read-only",
"--tmpfs",
"/tmp:rw,size=16m",
"-v",
str(ROOT / "research/load.js") + ":/script/load.js:ro",
"-v",
str(session) + ":/input/session.json:ro",
"-v",
str(logs) + ":/output",
"-e",
"K6_NO_USAGE_REPORT=true",
"-e",
"DURATION=" + str(duration),
"-e",
"WARMUP=" + str(warmup),
"-e",
"CONCURRENCY=" + str(config["concurrency"]),
K6,
"run",
"--quiet",
"/script/load.js",
],
check=True,
capture_output=True,
text=True,
)
generator = r.stdout.strip()
ids = dc("ps", "-q", env=env).stdout.split() + [generator]
sampler = threading.Thread(target=samples, args=(stop, ids, resources), daemon=True)
sampler.start()
result = subprocess.run(
["docker", "wait", generator],
check=True,
capture_output=True,
text=True,
timeout=duration + warmup + 35,
)
if result.stdout.strip() != "0":
raise RuntimeError("Load generator failed")
raw = json.loads((logs / "k6.json").read_text())
finally:
stop.set()
if "sampler" in locals():
sampler.join(timeout=6)
if generator:
subprocess.run(["docker", "rm", "-f", generator], capture_output=True, check=True)
sql_stats = [
dict(row)
for row in conn.execute(
"SELECT query,calls,total_exec_time,mean_exec_time,rows FROM pg_stat_statements "
"WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) "
"ORDER BY calls DESC LIMIT 30"
).fetchall()
]
metrics = raw["metrics"]
requests, successful = count(metrics, "measured_requests"), count(metrics, "measured_successes")
latency = metrics.get("successful_latency", {}).get("values", {})
final_rows = conn.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]
key_rows = conn.execute("SELECT count(*) AS n FROM message_requests").fetchone()["n"]
assert final_rows - 10000 == key_rows, "Message/result transaction invariant failed"
assert key_rows >= count(metrics, "all_confirmed_writes"), "Confirmed writes missing"
summary = {
"configuration": config,
"requests": requests,
"successes": successful,
"errors": count(metrics, "measured_errors"),
"error_fraction": (requests - successful) / requests,
"successful_rps": successful / raw["profile"]["measurement_wall_seconds"],
"p50_ms": latency.get("med"),
"p95_ms": latency.get("p(95)"),
"p99_ms": latency.get("p(99)"),
"final_messages": final_rows,
"idempotency_results": key_rows,
"resource_samples": resources,
"sql_stats": sql_stats,
"raw_k6": raw,
}
output.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
print(
json.dumps(
{
"trial": output.name,
"rps": round(summary["successful_rps"], 2),
"p95_ms": round(summary["p95_ms"], 2),
"errors": summary["errors"],
}
),
flush=True,
)
def matrix(pilot):
if pilot:
return [
{
"series": "pilot",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": mode,
"repeat": 1,
}
for mode in (False, True)
]
values = []
for repeat in range(1, 4):
for series in ("fixed", "growing"):
for replicas in (1, 2, 4):
values.append(
{
"series": series,
"replicas": replicas,
"cpu_each": 2 / replicas if series == "fixed" else 1,
"memory_mib_each": 1024 // replicas if series == "fixed" else 512,
"concurrency": 32,
"optimized": False,
"repeat": repeat,
}
)
for vus in (8, 96):
values.append(
{
"series": "saturation",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": vus,
"optimized": False,
"repeat": repeat,
}
)
values.append(
{
"series": "improvement",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": True,
"repeat": repeat,
}
)
for quota in (0.1, 0.25):
values.append(
{
"series": "database",
"replicas": 2,
"cpu_each": 1,
"memory_mib_each": 512,
"concurrency": 32,
"optimized": True,
"postgres_cpu": quota,
"repeat": repeat,
}
)
random.Random(2301).shuffle(values)
return values
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, default=ROOT / "evidence/scaling")
parser.add_argument("--duration", type=int, default=15)
parser.add_argument("--warmup", type=int, default=5)
parser.add_argument("--pilot", action="store_true")
args = parser.parse_args()
if args.output.exists():
raise SystemExit("Choose a new output directory")
args.output.mkdir(parents=True)
env = {
**os.environ,
"PUBLIC_PORT": "18083",
"TEST_PG_PORT": "55434",
"TEST_CACHE_PORT": "56381",
"API_REPLICAS": "2",
"SESSION_TTL_SECONDS": "3600",
"NO_PROXY": "localhost,127.0.0.1",
"no_proxy": "localhost,127.0.0.1",
}
# Separate project name and published DB port prevent touching the main lab3 data.
dc("up", "-d", "--no-build", "--wait", env=env)
password = (ROOT / "secrets/postgres_password").read_text().strip()
dsn = make_conninfo(
host="127.0.0.1",
port=55434,
dbname="messenger",
user="messenger",
password=password,
options="-c timezone=UTC",
connect_timeout=3,
)
metadata = {
"started_at": datetime.now(UTC).isoformat(),
"project": PROJECT,
"source_sha256": {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "src/messenger").glob("*.py")
},
"k6_image": K6,
"docker_host": json.loads(
subprocess.check_output(
["docker", "info", "--format", '{"cpu":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'],
text=True,
)
),
"api_image": subprocess.check_output(
["docker", "image", "inspect", "mtusi-messenger:lab3", "--format", "{{.Id}}"], text=True
).strip(),
"client_architecture": platform.machine(),
"trials": matrix(args.pilot),
}
try:
for index, config in enumerate(metadata["trials"], 1):
run_trial(
config,
env,
args.output / f"trial-{index:02d}.json",
dsn,
args.duration,
args.warmup,
)
assert metadata["source_sha256"] == {
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
for p in (ROOT / "src/messenger").glob("*.py")
}, "Runtime source changed mid-run"
metadata["finished_at"] = datetime.now(UTC).isoformat()
(args.output / "manifest.json").write_text(json.dumps(metadata, indent=2) + "\n")
print("PASS: isolated scaling matrix completed", flush=True)
finally:
dc("down", env=env) # Keeps this project's volume; never touches the main project.
if __name__ == "__main__":
main()