forked from mtusi2026/mtusi2026lab3
353 lines
12 KiB
Python
353 lines
12 KiB
Python
"""Изолированная матрица масштабирования; рабочие volumes не используются."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import platform
|
|
import socket
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def command(args, env):
|
|
r = subprocess.run(args, cwd=ROOT, env=env, capture_output=True, text=True)
|
|
if r.returncode:
|
|
raise RuntimeError(r.stderr.replace(str(ROOT), "<repository>"))
|
|
return r.stdout.strip()
|
|
|
|
|
|
def compose(env, *args):
|
|
return command(["docker", "compose", *args], env)
|
|
|
|
|
|
def percentile(values, p):
|
|
data = sorted(values)
|
|
return data[min(len(data) - 1, int((len(data) - 1) * p))] if data else None
|
|
|
|
|
|
async def load(base, token, chat, concurrency, duration):
|
|
headers = {"Authorization": "Bearer " + token}
|
|
rows = []
|
|
start = time.perf_counter()
|
|
async with httpx.AsyncClient(
|
|
base_url=base,
|
|
trust_env=False,
|
|
timeout=8,
|
|
limits=httpx.Limits(max_connections=concurrency),
|
|
) as c:
|
|
|
|
async def work(index):
|
|
sequence = index
|
|
while time.perf_counter() - start < duration:
|
|
at = time.perf_counter()
|
|
try:
|
|
if sequence % 5 == 0:
|
|
r = await c.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={
|
|
**headers,
|
|
"Idempotency-Key": uuid4().hex,
|
|
},
|
|
json={"text": "Scaling synthetic message 🌍"},
|
|
)
|
|
else:
|
|
r = await c.get(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers=headers,
|
|
params={"limit": 20},
|
|
)
|
|
status = r.status_code
|
|
instance = r.headers.get("x-instance-id")
|
|
except httpx.HTTPError:
|
|
status, instance = 0, None
|
|
rows.append(
|
|
[
|
|
round((at - start) * 1000, 3),
|
|
round((time.perf_counter() - at) * 1000, 3),
|
|
status,
|
|
instance,
|
|
]
|
|
)
|
|
sequence += 1
|
|
|
|
await asyncio.gather(*(work(i) for i in range(concurrency)))
|
|
elapsed = time.perf_counter() - start
|
|
good = [r[1] for r in rows if r[2] in (200, 201)]
|
|
return {
|
|
"elapsed_seconds": elapsed,
|
|
"successful_rps": len(good) / elapsed,
|
|
"success": len(good),
|
|
"errors": len(rows) - len(good),
|
|
"error_ratio": (len(rows) - len(good)) / max(1, len(rows)),
|
|
"p50_ms": percentile(good, 0.5),
|
|
"p95_ms": percentile(good, 0.95),
|
|
"p99_ms": percentile(good, 0.99),
|
|
"requests": rows,
|
|
}
|
|
|
|
|
|
def cpu_counters(env):
|
|
ids = compose(env, "ps", "-q", "api").split()
|
|
code = (
|
|
"import json,pathlib; print(json.dumps({p: "
|
|
"pathlib.Path('/sys/fs/cgroup/'+p).read_text() for p in "
|
|
"['cpu.stat','memory.current','memory.peak']}))"
|
|
)
|
|
return {
|
|
cid[:12]: json.loads(
|
|
command(["docker", "exec", cid, "python", "-c", code], env)
|
|
)
|
|
for cid in ids
|
|
}
|
|
|
|
|
|
def sample(env, stop, output):
|
|
while not stop.is_set():
|
|
try:
|
|
ids = compose(env, "ps", "-q").split()
|
|
raw = command(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{json .}}",
|
|
*ids,
|
|
],
|
|
env,
|
|
)
|
|
output.append(
|
|
{
|
|
"at": time.time(),
|
|
"containers": [json.loads(s) for s in raw.splitlines()],
|
|
}
|
|
)
|
|
except Exception as exc:
|
|
output.append({"error": type(exc).__name__})
|
|
stop.wait(0.25)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--output", default="evidence/scaling-repeat")
|
|
p.add_argument("--duration", type=float, default=6)
|
|
p.add_argument("--warmup", type=float, default=2)
|
|
p.add_argument("--only-improvement", action="store_true")
|
|
a = p.parse_args()
|
|
out = Path(a.output)
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
project = "lab3-scaling-" + uuid4().hex[:8]
|
|
env = {
|
|
**os.environ,
|
|
"COMPOSE_PROJECT_NAME": project,
|
|
"PUBLIC_PORT": str(port),
|
|
}
|
|
base = f"http://127.0.0.1:{port}"
|
|
meta = {
|
|
"source_commit": command(["git", "rev-parse", "HEAD"], env),
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"host": platform.platform(),
|
|
"docker": json.loads(
|
|
command(["docker", "info", "--format", "{{json .}}"], env)
|
|
),
|
|
"duration": a.duration,
|
|
"warmup": a.warmup,
|
|
"profile": "80% GET history limit20; 20% POST unique key; closed-loop",
|
|
"complete": False,
|
|
"cells": [],
|
|
}
|
|
# Полный docker info не нужен: сохраняется только воспроизводимое железо.
|
|
meta["docker"] = {
|
|
k: meta["docker"][k]
|
|
for k in [
|
|
"ServerVersion",
|
|
"NCPU",
|
|
"MemTotal",
|
|
"Architecture",
|
|
"OperatingSystem",
|
|
]
|
|
}
|
|
cases = []
|
|
if not a.only_improvement:
|
|
for series in ["fixed", "growing"]:
|
|
for n in [1, 2, 4]:
|
|
for concurrency in [1, 8, 32]:
|
|
for repeat in range(1, 3):
|
|
cases.append((series, n, concurrency, repeat, 10))
|
|
for repeat in range(1, 4):
|
|
# Чередование снижает систематический дрейф температуры/кэша.
|
|
for pool in [1, 10] if repeat % 2 else [10, 1]:
|
|
cases.append(("pool", 2, 32, repeat, pool))
|
|
revision = command(
|
|
[
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
env.get("API_IMAGE", "mtusi-messenger:lab3"),
|
|
"--format",
|
|
'{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
|
],
|
|
env,
|
|
)
|
|
meta["runtime_image_revision"] = revision
|
|
assert (
|
|
command(
|
|
[
|
|
"git",
|
|
"diff",
|
|
revision,
|
|
"--",
|
|
"src",
|
|
"migrations",
|
|
"pyproject.toml",
|
|
"uv.lock",
|
|
"Dockerfile",
|
|
],
|
|
env,
|
|
)
|
|
== ""
|
|
)
|
|
meta["runtime_equal_to_source_commit"] = True
|
|
current = None
|
|
try:
|
|
for number, (series, n, c, rep, pool) in enumerate(cases, 1):
|
|
cpus = 1 / n if series == "fixed" else 0.5
|
|
memory = int(1024 / n) if series == "fixed" else 256
|
|
env.update(
|
|
API_REPLICAS=str(n),
|
|
API_CPUS=str(cpus),
|
|
API_MEMORY=f"{memory}m",
|
|
DB_POOL_SIZE=str(pool),
|
|
)
|
|
configuration = (n, cpus, memory, pool)
|
|
if configuration != current:
|
|
compose(env, "up", "-d", "--no-build", "--wait")
|
|
current = configuration
|
|
time.sleep(3)
|
|
# Только новая БД изолированного исследовательского проекта.
|
|
assert env["COMPOSE_PROJECT_NAME"].startswith("lab3-scaling-")
|
|
compose(
|
|
env,
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-c",
|
|
"TRUNCATE idempotency,messages,members,chats,sessions,users",
|
|
)
|
|
with httpx.Client(
|
|
base_url=base, trust_env=False, timeout=15
|
|
) as client:
|
|
username = "scale_" + uuid4().hex[:20]
|
|
password = uuid4().hex
|
|
r = client.post(
|
|
"/api/v1/users",
|
|
json={
|
|
"username": username,
|
|
"display_name": "Synthetic",
|
|
"password": password,
|
|
},
|
|
)
|
|
assert r.status_code == 201
|
|
user = r.json()
|
|
r = client.post(
|
|
"/api/v1/auth/sessions",
|
|
auth=httpx.BasicAuth(username, password),
|
|
)
|
|
assert r.status_code == 201
|
|
token = r.json()["token"]
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers={"Authorization": "Bearer " + token},
|
|
json={"title": "Scaling", "member_ids": [user["id"]]},
|
|
)
|
|
assert r.status_code == 201
|
|
chat = r.json()["id"]
|
|
for i in range(100):
|
|
r = client.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={
|
|
"Authorization": "Bearer " + token,
|
|
"Idempotency-Key": uuid4().hex,
|
|
},
|
|
json={"text": f"Seed {i}"},
|
|
)
|
|
assert r.status_code == 201
|
|
asyncio.run(load(base, token, chat, c, a.warmup))
|
|
counters_before = cpu_counters(env)
|
|
samples = []
|
|
stop = threading.Event()
|
|
thread = threading.Thread(target=sample, args=(env, stop, samples))
|
|
thread.start()
|
|
try:
|
|
result = asyncio.run(load(base, token, chat, c, a.duration))
|
|
finally:
|
|
stop.set()
|
|
thread.join()
|
|
counters_after = cpu_counters(env)
|
|
ids = compose(env, "ps", "-q").split()
|
|
inspect = json.loads(command(["docker", "inspect", *ids], env))
|
|
result.update(
|
|
series=series,
|
|
replicas=n,
|
|
concurrency=c,
|
|
repeat=rep,
|
|
pool_size=pool,
|
|
api_cpu_each=cpus,
|
|
api_memory_mib_each=memory,
|
|
dataset_initial_messages=100,
|
|
samples=samples,
|
|
cgroup_before=counters_before,
|
|
cgroup_after=counters_after,
|
|
actual_limits=[
|
|
{
|
|
"role": x["Config"]["Labels"][
|
|
"com.docker.compose.service"
|
|
],
|
|
"cpu_nano": x["HostConfig"]["NanoCpus"],
|
|
"memory": x["HostConfig"]["Memory"],
|
|
}
|
|
for x in inspect
|
|
],
|
|
)
|
|
filename = f"{number:02}-{series}-n{n}-c{c}-r{rep}-p{pool}.json"
|
|
(out / filename).write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
meta["cells"].append(filename)
|
|
(out / "run.json").write_text(
|
|
json.dumps(meta, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(
|
|
f"{number}/{len(cases)} {filename}: {result['successful_rps']:.1f} RPS, {result['errors']} errors",
|
|
flush=True,
|
|
)
|
|
meta["complete"] = True
|
|
finally:
|
|
(out / "run.json").write_text(
|
|
json.dumps(meta, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
compose(env, "down", "--volumes", "--remove-orphans")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|