Files
mtusi2026lab3/research/compare.py
T

174 lines
5.7 KiB
Python

"""Чередующееся сравнение двух собственных образов на отдельном стенде."""
import argparse
import asyncio
import json
import os
import socket
import threading
import time
from datetime import UTC, datetime
from pathlib import Path
from uuid import uuid4
import httpx
from research.run import command, compose, cpu_counters, load, sample
from scripts.stand import actor, auth
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--before", required=True)
parser.add_argument("--after", required=True)
parser.add_argument("--output", required=True)
args = parser.parse_args()
out = Path(args.output)
out.mkdir(parents=True, exist_ok=False)
with socket.socket() as sock:
sock.bind(("127.0.0.1", 0))
port = sock.getsockname()[1]
project = "lab3-compare-" + uuid4().hex[:8]
env = {
**os.environ,
"COMPOSE_PROJECT_NAME": project,
"PUBLIC_PORT": str(port),
"API_REPLICAS": "4",
"API_CPUS": "1",
"API_MEMORY": "384m",
"DB_POOL_SIZE": "10",
}
report = {
"date": datetime.now(UTC).isoformat(),
"complete": False,
"profile": "80% GET limit20;20% POST; C16; 100 seed; warmup2s/measure12s; four API each1CPU/384MiB; PostgreSQL 0.1 vs 1CPU",
"images": {},
"cells": [],
}
for name, image in [("before", args.before), ("after", args.after)]:
info = json.loads(command(["docker", "image", "inspect", image], env))[
0
]
report["images"][name] = {
"image": image,
"digest": info["Id"],
"revision": info["Config"]["Labels"][
"org.opencontainers.image.revision"
],
}
try:
number = 0
cases = [
(repeat, version, pg_cpu)
for repeat in range(1, 4)
for pg_cpu in [0.1, 1.0]
for version in (
["before", "after"] if repeat % 2 else ["after", "before"]
)
]
for repeat, version, pg_cpu in cases:
number += 1
env["API_IMAGE"] = (
args.before if version == "before" else args.after
)
compose(env, "up", "-d", "--no-build", "--wait")
pg_cid = compose(env, "ps", "-q", "postgres")
command(["docker", "update", "--cpus", str(pg_cpu), pg_cid], env)
time.sleep(3)
assert project.startswith("lab3-compare-")
compose(
env,
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-v",
"ON_ERROR_STOP=1",
"-c",
"TRUNCATE idempotency,messages,members,chats,sessions,users",
)
base = f"http://127.0.0.1:{port}"
with httpx.Client(
base_url=base, trust_env=False, timeout=15
) as client:
user, _, session = actor(client)
r = client.post(
"/api/v1/chats",
headers=auth(session),
json={
"title": "Controlled comparison",
"member_ids": [user["id"]],
},
)
assert r.status_code == 201
chat = r.json()["id"]
for i in range(100):
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={
**auth(session),
"Idempotency-Key": uuid4().hex,
},
json={"text": f"Seed {i}"},
)
assert r.status_code == 201
asyncio.run(load(base, session["token"], chat, 16, 2))
counters = cpu_counters(env)
pg_before = command(
["docker", "exec", pg_cid, "cat", "/sys/fs/cgroup/cpu.stat"],
env,
)
samples = []
stop = threading.Event()
thread = threading.Thread(target=sample, args=(env, stop, samples))
thread.start()
try:
result = asyncio.run(
load(base, session["token"], chat, 16, 12)
)
finally:
stop.set()
thread.join()
result.update(
version=version,
repeat=repeat,
samples=samples,
pg_cpu_limit=pg_cpu,
pg_cpu_before=pg_before,
pg_cpu_after=command(
[
"docker",
"exec",
pg_cid,
"cat",
"/sys/fs/cgroup/cpu.stat",
],
env,
),
cgroup_before=counters,
cgroup_after=cpu_counters(env),
)
name = f"{number:02}-{version}-pg{pg_cpu}-r{repeat}.json"
(out / name).write_text(
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
)
report["cells"].append(name)
print(
f"{name}: {result['successful_rps']:.1f} RPS; {result['errors']} errors",
flush=True,
)
report["complete"] = True
finally:
(out / "run.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
compose(env, "down", "--volumes", "--remove-orphans")
if __name__ == "__main__":
main()