415 lines
14 KiB
Python
415 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Repeatable scaling matrix on an isolated Compose project and fresh data."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import psycopg
|
|
from psycopg.conninfo import make_conninfo
|
|
from psycopg.rows import dict_row
|
|
|
|
from messenger.cursor import encode_cursor
|
|
from messenger.repository import stamp
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PROJECT = "mtusi-lab3-research"
|
|
K6 = "grafana/k6@sha256:3ddc8b1a33a2c3d8edc6e99b6a762ae36cba08788463458f5e6a7703e14eb77d"
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure # noqa: E402
|
|
|
|
|
|
def dc(*args, env):
|
|
return subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"-p",
|
|
PROJECT,
|
|
"--env-file",
|
|
".env",
|
|
"-f",
|
|
"compose.yaml",
|
|
"-f",
|
|
"compose.test.yaml",
|
|
*args,
|
|
],
|
|
cwd=ROOT,
|
|
env=env,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
|
|
|
|
def count(metrics, key):
|
|
return metrics.get(key, {}).get("values", {}).get("count", 0)
|
|
|
|
|
|
def wait_api(client, expected):
|
|
seen, deadline = set(), time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response = client.call("GET", "/health/ready", schema="Health")
|
|
seen.add(response["headers"]["X-Instance-Id"])
|
|
if set(expected) <= seen:
|
|
return sorted(expected)
|
|
except Failure:
|
|
pass
|
|
time.sleep(0.1)
|
|
raise RuntimeError("Research APIs failed to become ready")
|
|
|
|
|
|
def samples(stop, container_ids, result):
|
|
while not stop.is_set():
|
|
try:
|
|
r = subprocess.run(
|
|
["docker", "stats", "--no-stream", "--format", "{{json .}}", *container_ids],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
timeout=5,
|
|
)
|
|
result.append(
|
|
{
|
|
"at": datetime.now(UTC).isoformat(),
|
|
"containers": [
|
|
{k: value[k] for k in ("ID", "Name", "CPUPerc", "MemUsage", "MemPerc")}
|
|
for value in map(json.loads, r.stdout.splitlines())
|
|
],
|
|
}
|
|
)
|
|
except OSError, ValueError, KeyError, subprocess.SubprocessError:
|
|
result.append({"error": "resource_sample_unavailable"})
|
|
stop.wait(0.25)
|
|
|
|
|
|
def fixture(conn, client):
|
|
# This connection only belongs to PROJECT's separate PostgreSQL container.
|
|
conn.execute("TRUNCATE users,chats,chat_members,messages,sessions,message_requests CASCADE")
|
|
alice, _, _, chat = client.bootstrap()
|
|
conn.execute(
|
|
"INSERT INTO messages (id,chat_id,sender_id,text,created_at) "
|
|
"SELECT md5(%s || g::text)::uuid,%s,%s,repeat('m',128),"
|
|
"'2026-10-17T00:00:00Z'::timestamptz + g*interval '1 microsecond' "
|
|
"FROM generate_series(1,10000) g",
|
|
(chat["id"], chat["id"], alice["id"]),
|
|
)
|
|
conn.execute("VACUUM ANALYZE messages")
|
|
row = conn.execute(
|
|
"SELECT created_at,id FROM messages WHERE chat_id=%s "
|
|
"ORDER BY created_at,id OFFSET 9000 LIMIT 1",
|
|
(chat["id"],),
|
|
).fetchone()
|
|
cursor = encode_cursor("messages:" + chat["id"], (stamp(row["created_at"]), str(row["id"])))
|
|
return {"token": alice["token"], "chat": chat["id"], "cursor": cursor, "run": uuid4().hex}
|
|
|
|
|
|
def run_trial(config, env, output, dsn, duration, warmup):
|
|
env = {
|
|
**env,
|
|
"API_REPLICAS": str(config["replicas"]),
|
|
"POSTGRES_CPU": str(config.get("postgres_cpu", 1)),
|
|
"API_CPU": str(config["cpu_each"]),
|
|
"API_MEMORY": str(config["memory_mib_each"]) + "m",
|
|
"DATABASE_READ_AUTOCOMMIT": str(int(config["optimized"])),
|
|
}
|
|
dc("up", "-d", "--no-build", "--wait", env=env)
|
|
# PostgreSQL CPU changes recreate its container; never reuse that old socket.
|
|
with psycopg.connect(dsn, autocommit=True, row_factory=dict_row) as conn:
|
|
conn.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements")
|
|
measure(config, env, output, conn, duration, warmup)
|
|
|
|
|
|
def measure(config, env, output, conn, duration, warmup):
|
|
client = Client("http://127.0.0.1:18083", timeout=5)
|
|
ids = dc("ps", "-q", "api", env=env).stdout.split()
|
|
assert len(ids) == config["replicas"]
|
|
aliases = []
|
|
for container in ids:
|
|
code = (
|
|
"import urllib.request; r=urllib.request.urlopen("
|
|
"'http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"
|
|
)
|
|
aliases.append(
|
|
subprocess.check_output(
|
|
["docker", "exec", container, "python", "-c", code], text=True
|
|
).strip()
|
|
)
|
|
aliases = wait_api(client, aliases)
|
|
dc("exec", "-T", "valkey", "valkey-cli", "FLUSHDB", env=env)
|
|
data = fixture(conn, client)
|
|
data["instances"] = aliases
|
|
conn.execute("SELECT pg_stat_statements_reset()")
|
|
resources, stop, generator = [], threading.Event(), None
|
|
with tempfile.TemporaryDirectory(prefix="lab3-load-") as temporary:
|
|
temporary = Path(temporary)
|
|
session = temporary / "session.json"
|
|
session.write_text(json.dumps(data))
|
|
os.chmod(session, 0o600)
|
|
logs = temporary / "out"
|
|
logs.mkdir()
|
|
try:
|
|
r = subprocess.run(
|
|
[
|
|
"docker",
|
|
"run",
|
|
"-d",
|
|
"--network",
|
|
PROJECT + "_backend",
|
|
"--cpus",
|
|
"2",
|
|
"--memory",
|
|
"384m",
|
|
"--memory-swap",
|
|
"384m",
|
|
"--user",
|
|
str(os.getuid()) + ":" + str(os.getgid()),
|
|
"--read-only",
|
|
"--tmpfs",
|
|
"/tmp:rw,size=16m",
|
|
"-v",
|
|
str(ROOT / "research/load.js") + ":/script/load.js:ro",
|
|
"-v",
|
|
str(session) + ":/input/session.json:ro",
|
|
"-v",
|
|
str(logs) + ":/output",
|
|
"-e",
|
|
"K6_NO_USAGE_REPORT=true",
|
|
"-e",
|
|
"DURATION=" + str(duration),
|
|
"-e",
|
|
"WARMUP=" + str(warmup),
|
|
"-e",
|
|
"CONCURRENCY=" + str(config["concurrency"]),
|
|
K6,
|
|
"run",
|
|
"--quiet",
|
|
"/script/load.js",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
generator = r.stdout.strip()
|
|
ids = dc("ps", "-q", env=env).stdout.split() + [generator]
|
|
sampler = threading.Thread(target=samples, args=(stop, ids, resources), daemon=True)
|
|
sampler.start()
|
|
result = subprocess.run(
|
|
["docker", "wait", generator],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=duration + warmup + 35,
|
|
)
|
|
if result.stdout.strip() != "0":
|
|
raise RuntimeError("Load generator failed")
|
|
raw = json.loads((logs / "k6.json").read_text())
|
|
finally:
|
|
stop.set()
|
|
if "sampler" in locals():
|
|
sampler.join(timeout=6)
|
|
if generator:
|
|
subprocess.run(["docker", "rm", "-f", generator], capture_output=True, check=True)
|
|
sql_stats = [
|
|
dict(row)
|
|
for row in conn.execute(
|
|
"SELECT query,calls,total_exec_time,mean_exec_time,rows FROM pg_stat_statements "
|
|
"WHERE dbid=(SELECT oid FROM pg_database WHERE datname=current_database()) "
|
|
"ORDER BY calls DESC LIMIT 30"
|
|
).fetchall()
|
|
]
|
|
metrics = raw["metrics"]
|
|
requests, successful = count(metrics, "measured_requests"), count(metrics, "measured_successes")
|
|
latency = metrics.get("successful_latency", {}).get("values", {})
|
|
final_rows = conn.execute("SELECT count(*) AS n FROM messages").fetchone()["n"]
|
|
key_rows = conn.execute("SELECT count(*) AS n FROM message_requests").fetchone()["n"]
|
|
assert final_rows - 10000 == key_rows, "Message/result transaction invariant failed"
|
|
assert key_rows >= count(metrics, "all_confirmed_writes"), "Confirmed writes missing"
|
|
summary = {
|
|
"configuration": config,
|
|
"requests": requests,
|
|
"successes": successful,
|
|
"errors": count(metrics, "measured_errors"),
|
|
"error_fraction": (requests - successful) / requests,
|
|
"successful_rps": successful / raw["profile"]["measurement_wall_seconds"],
|
|
"p50_ms": latency.get("med"),
|
|
"p95_ms": latency.get("p(95)"),
|
|
"p99_ms": latency.get("p(99)"),
|
|
"final_messages": final_rows,
|
|
"idempotency_results": key_rows,
|
|
"resource_samples": resources,
|
|
"sql_stats": sql_stats,
|
|
"raw_k6": raw,
|
|
}
|
|
output.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"trial": output.name,
|
|
"rps": round(summary["successful_rps"], 2),
|
|
"p95_ms": round(summary["p95_ms"], 2),
|
|
"errors": summary["errors"],
|
|
}
|
|
),
|
|
flush=True,
|
|
)
|
|
|
|
|
|
def matrix(pilot):
|
|
if pilot:
|
|
return [
|
|
{
|
|
"series": "pilot",
|
|
"replicas": 2,
|
|
"cpu_each": 1,
|
|
"memory_mib_each": 512,
|
|
"concurrency": 32,
|
|
"optimized": mode,
|
|
"repeat": 1,
|
|
}
|
|
for mode in (False, True)
|
|
]
|
|
values = []
|
|
for repeat in range(1, 4):
|
|
for series in ("fixed", "growing"):
|
|
for replicas in (1, 2, 4):
|
|
values.append(
|
|
{
|
|
"series": series,
|
|
"replicas": replicas,
|
|
"cpu_each": 2 / replicas if series == "fixed" else 1,
|
|
"memory_mib_each": 1024 // replicas if series == "fixed" else 512,
|
|
"concurrency": 32,
|
|
"optimized": False,
|
|
"repeat": repeat,
|
|
}
|
|
)
|
|
for vus in (8, 96):
|
|
values.append(
|
|
{
|
|
"series": "saturation",
|
|
"replicas": 2,
|
|
"cpu_each": 1,
|
|
"memory_mib_each": 512,
|
|
"concurrency": vus,
|
|
"optimized": False,
|
|
"repeat": repeat,
|
|
}
|
|
)
|
|
values.append(
|
|
{
|
|
"series": "improvement",
|
|
"replicas": 2,
|
|
"cpu_each": 1,
|
|
"memory_mib_each": 512,
|
|
"concurrency": 32,
|
|
"optimized": True,
|
|
"repeat": repeat,
|
|
}
|
|
)
|
|
for quota in (0.1, 0.25):
|
|
values.append(
|
|
{
|
|
"series": "database",
|
|
"replicas": 2,
|
|
"cpu_each": 1,
|
|
"memory_mib_each": 512,
|
|
"concurrency": 32,
|
|
"optimized": True,
|
|
"postgres_cpu": quota,
|
|
"repeat": repeat,
|
|
}
|
|
)
|
|
random.Random(2301).shuffle(values)
|
|
return values
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--output", type=Path, default=ROOT / "evidence/scaling")
|
|
parser.add_argument("--duration", type=int, default=15)
|
|
parser.add_argument("--warmup", type=int, default=5)
|
|
parser.add_argument("--pilot", action="store_true")
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
raise SystemExit("Choose a new output directory")
|
|
args.output.mkdir(parents=True)
|
|
env = {
|
|
**os.environ,
|
|
"PUBLIC_PORT": "18083",
|
|
"TEST_PG_PORT": "55434",
|
|
"TEST_CACHE_PORT": "56381",
|
|
"API_REPLICAS": "2",
|
|
"SESSION_TTL_SECONDS": "3600",
|
|
"NO_PROXY": "localhost,127.0.0.1",
|
|
"no_proxy": "localhost,127.0.0.1",
|
|
}
|
|
# Separate project name and published DB port prevent touching the main lab3 data.
|
|
dc("up", "-d", "--no-build", "--wait", env=env)
|
|
password = (ROOT / "secrets/postgres_password").read_text().strip()
|
|
dsn = make_conninfo(
|
|
host="127.0.0.1",
|
|
port=55434,
|
|
dbname="messenger",
|
|
user="messenger",
|
|
password=password,
|
|
options="-c timezone=UTC",
|
|
connect_timeout=3,
|
|
)
|
|
metadata = {
|
|
"started_at": datetime.now(UTC).isoformat(),
|
|
"project": PROJECT,
|
|
"source_sha256": {
|
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in (ROOT / "src/messenger").glob("*.py")
|
|
},
|
|
"k6_image": K6,
|
|
"docker_host": json.loads(
|
|
subprocess.check_output(
|
|
["docker", "info", "--format", '{"cpu":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'],
|
|
text=True,
|
|
)
|
|
),
|
|
"api_image": subprocess.check_output(
|
|
["docker", "image", "inspect", "mtusi-messenger:lab3", "--format", "{{.Id}}"], text=True
|
|
).strip(),
|
|
"client_architecture": platform.machine(),
|
|
"trials": matrix(args.pilot),
|
|
}
|
|
try:
|
|
for index, config in enumerate(metadata["trials"], 1):
|
|
run_trial(
|
|
config,
|
|
env,
|
|
args.output / f"trial-{index:02d}.json",
|
|
dsn,
|
|
args.duration,
|
|
args.warmup,
|
|
)
|
|
assert metadata["source_sha256"] == {
|
|
p.name: hashlib.sha256(p.read_bytes()).hexdigest()
|
|
for p in (ROOT / "src/messenger").glob("*.py")
|
|
}, "Runtime source changed mid-run"
|
|
metadata["finished_at"] = datetime.now(UTC).isoformat()
|
|
(args.output / "manifest.json").write_text(json.dumps(metadata, indent=2) + "\n")
|
|
print("PASS: isolated scaling matrix completed", flush=True)
|
|
finally:
|
|
dc("down", env=env) # Keeps this project's volume; never touches the main project.
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|