Files
mtusi2026lab3/research/run.py
T

353 lines
12 KiB
Python

"""Изолированная матрица масштабирования; рабочие volumes не используются."""
import argparse
import asyncio
import json
import os
import platform
import socket
import subprocess
import threading
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import httpx
ROOT = Path(__file__).resolve().parents[1]
def command(args, env):
r = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True)
if r.returncode:
raise RuntimeError(r.stderr.replace(str(ROOT), "<repository>"))
return r.stdout.strip()
def compose(env, *args):
return command(["docker", "compose", *args], env)
def percentile(values, p):
data = sorted(values)
return data[min(len(data) - 1, int((len(data) - 1) * p))] if data else None
async def load(base, token, chat, concurrency, duration):
headers = {"Authorization": "Bearer " + token}
rows = []
start = time.perf_counter()
async with httpx.AsyncClient(
base_url=base,
trust_env=False,
timeout=8,
limits=httpx.Limits(max_connections=concurrency),
) as c:
async def work(index):
sequence = index
while time.perf_counter() - start < duration:
at = time.perf_counter()
try:
if sequence % 5 == 0:
r = await c.post(
f"/api/v1/chats/{chat}/messages",
headers={
**headers,
"Idempotency-Key": uuid4().hex,
},
json={"text": "Scaling synthetic message 🌍"},
)
else:
r = await c.get(
f"/api/v1/chats/{chat}/messages",
headers=headers,
params={"limit": 20},
)
status = r.status_code
instance = r.headers.get("x-instance-id")
except httpx.HTTPError:
status, instance = 0, None
rows.append(
[
round((at - start) * 1000, 3),
round((time.perf_counter() - at) * 1000, 3),
status,
instance,
]
)
sequence += 1
await asyncio.gather(*(work(i) for i in range(concurrency)))
elapsed = time.perf_counter() - start
good = [r[1] for r in rows if r[2] in (200, 201)]
return {
"elapsed_seconds": elapsed,
"successful_rps": len(good) / elapsed,
"success": len(good),
"errors": len(rows) - len(good),
"error_ratio": (len(rows) - len(good)) / max(1, len(rows)),
"p50_ms": percentile(good, 0.5),
"p95_ms": percentile(good, 0.95),
"p99_ms": percentile(good, 0.99),
"requests": rows,
}
def cpu_counters(env):
ids = compose(env, "ps", "-q", "api").split()
code = (
"import json,pathlib; print(json.dumps({p: "
"pathlib.Path('/sys/fs/cgroup/'+p).read_text() for p in "
"['cpu.stat','memory.current','memory.peak']}))"
)
return {
cid[:12]: json.loads(
command(["docker", "exec", cid, "python", "-c", code], env)
)
for cid in ids
}
def sample(env, stop, output):
while not stop.is_set():
try:
ids = compose(env, "ps", "-q").split()
raw = command(
[
"docker",
"stats",
"--no-stream",
"--format",
"{{json .}}",
*ids,
],
env,
)
output.append(
{
"at": time.time(),
"containers": [json.loads(s) for s in raw.splitlines()],
}
)
except Exception as exc:
output.append({"error": type(exc).__name__})
stop.wait(0.25)
def main():
p = argparse.ArgumentParser()
p.add_argument("--output", default="evidence/scaling-repeat")
p.add_argument("--duration", type=float, default=6)
p.add_argument("--warmup", type=float, default=2)
p.add_argument("--only-improvement", action="store_true")
a = p.parse_args()
out = Path(a.output)
out.mkdir(parents=True, exist_ok=False)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
project = "lab3-scaling-" + uuid4().hex[:8]
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"PUBLIC_PORT": str(port),
}
base = f"http://127.0.0.1:{port}"
meta = {
"source_commit": command(["git", "rev-parse", "HEAD"], env),
"date": datetime.now(UTC).isoformat(),
"host": platform.platform(),
"docker": json.loads(
command(["docker", "info", "--format", "{{json .}}"], env)
),
"duration": a.duration,
"warmup": a.warmup,
"profile": "80% GET history limit20; 20% POST unique key; closed-loop",
"complete": False,
"cells": [],
}
# Полный docker info не нужен: сохраняется только воспроизводимое железо.
meta["docker"] = {
k: meta["docker"][k]
for k in [
"ServerVersion",
"NCPU",
"MemTotal",
"Architecture",
"OperatingSystem",
]
}
cases = []
if not a.only_improvement:
for series in ["fixed", "growing"]:
for n in [1, 2, 4]:
for concurrency in [1, 8, 32]:
for repeat in range(1, 3):
cases.append((series, n, concurrency, repeat, 10))
for repeat in range(1, 4):
# Чередование снижает систематический дрейф температуры/кэша.
for pool in [1, 10] if repeat % 2 else [10, 1]:
cases.append(("pool", 2, 32, repeat, pool))
revision = command(
[
"docker",
"image",
"inspect",
env.get("API_IMAGE", "mtusi-messenger:lab3"),
"--format",
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
],
env,
)
meta["runtime_image_revision"] = revision
assert (
command(
[
"git",
"diff",
revision,
"--",
"src",
"migrations",
"pyproject.toml",
"uv.lock",
"Dockerfile",
],
env,
)
== ""
)
meta["runtime_equal_to_source_commit"] = True
current = None
try:
for number, (series, n, c, rep, pool) in enumerate(cases, 1):
cpus = 1 / n if series == "fixed" else 0.5
memory = int(1024 / n) if series == "fixed" else 256
env.update(
API_REPLICAS=str(n),
API_CPUS=str(cpus),
API_MEMORY=f"{memory}m",
DB_POOL_SIZE=str(pool),
)
configuration = (n, cpus, memory, pool)
if configuration != current:
compose(env, "up", "-d", "--no-build", "--wait")
current = configuration
time.sleep(3)
# Только новая БД изолированного исследовательского проекта.
assert env["COMPOSE_PROJECT_NAME"].startswith("lab3-scaling-")
compose(
env,
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
"-c",
"TRUNCATE idempotency,messages,members,chats,sessions,users",
)
with httpx.Client(
base_url=base, trust_env=False, timeout=15
) as client:
username = "scale_" + uuid4().hex[:20]
password = uuid4().hex
r = client.post(
"/api/v1/users",
json={
"username": username,
"display_name": "Synthetic",
"password": password,
},
)
assert r.status_code == 201
user = r.json()
r = client.post(
"/api/v1/auth/sessions",
auth=httpx.BasicAuth(username, password),
)
assert r.status_code == 201
token = r.json()["token"]
r = client.post(
"/api/v1/chats",
headers={"Authorization": "Bearer " + token},
json={"title": "Scaling", "member_ids": [user["id"]]},
)
assert r.status_code == 201
chat = r.json()["id"]
for i in range(100):
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={
"Authorization": "Bearer " + token,
"Idempotency-Key": uuid4().hex,
},
json={"text": f"Seed {i}"},
)
assert r.status_code == 201
asyncio.run(load(base, token, chat, c, a.warmup))
counters_before = cpu_counters(env)
samples = []
stop = threading.Event()
thread = threading.Thread(target=sample, args=(env, stop, samples))
thread.start()
try:
result = asyncio.run(load(base, token, chat, c, a.duration))
finally:
stop.set()
thread.join()
counters_after = cpu_counters(env)
ids = compose(env, "ps", "-q").split()
inspect = json.loads(command(["docker", "inspect", *ids], env))
result.update(
series=series,
replicas=n,
concurrency=c,
repeat=rep,
pool_size=pool,
api_cpu_each=cpus,
api_memory_mib_each=memory,
dataset_initial_messages=100,
samples=samples,
cgroup_before=counters_before,
cgroup_after=counters_after,
actual_limits=[
{
"role": x["Config"]["Labels"][
"com.docker.compose.service"
],
"cpu_nano": x["HostConfig"]["NanoCpus"],
"memory": x["HostConfig"]["Memory"],
}
for x in inspect
],
)
filename = f"{number:02}-{series}-n{n}-c{c}-r{rep}-p{pool}.json"
(out / filename).write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
meta["cells"].append(filename)
(out / "run.json").write_text(
json.dumps(meta, ensure_ascii=False, indent=2) + "\n"
)
print(
f"{number}/{len(cases)} {filename}: {result['successful_rps']:.1f} RPS, {result['errors']} errors",
flush=True,
)
meta["complete"] = True
finally:
(out / "run.json").write_text(
json.dumps(meta, ensure_ascii=False, indent=2) + "\n"
)
compose(env, "down", "--volumes", "--remove-orphans")
if __name__ == "__main__":
main()