174 lines
5.7 KiB
Python
174 lines
5.7 KiB
Python
"""Чередующееся сравнение двух собственных образов на отдельном стенде."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import socket
|
|
import threading
|
|
import time
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import httpx
|
|
|
|
from research.run import command, compose, cpu_counters, load, sample
|
|
from scripts.stand import actor, auth
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--before", required=True)
|
|
parser.add_argument("--after", required=True)
|
|
parser.add_argument("--output", required=True)
|
|
args = parser.parse_args()
|
|
out = Path(args.output)
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
with socket.socket() as sock:
|
|
sock.bind(("127.0.0.1", 0))
|
|
port = sock.getsockname()[1]
|
|
project = "lab3-compare-" + uuid4().hex[:8]
|
|
env = {
|
|
**os.environ,
|
|
"COMPOSE_PROJECT_NAME": project,
|
|
"PUBLIC_PORT": str(port),
|
|
"API_REPLICAS": "4",
|
|
"API_CPUS": "1",
|
|
"API_MEMORY": "384m",
|
|
"DB_POOL_SIZE": "10",
|
|
}
|
|
report = {
|
|
"date": datetime.now(UTC).isoformat(),
|
|
"complete": False,
|
|
"profile": "80% GET limit20;20% POST; C16; 100 seed; warmup2s/measure12s; four API each1CPU/384MiB; PostgreSQL 0.1 vs 1CPU",
|
|
"images": {},
|
|
"cells": [],
|
|
}
|
|
for name, image in [("before", args.before), ("after", args.after)]:
|
|
info = json.loads(command(["docker", "image", "inspect", image], env))[
|
|
0
|
|
]
|
|
report["images"][name] = {
|
|
"image": image,
|
|
"digest": info["Id"],
|
|
"revision": info["Config"]["Labels"][
|
|
"org.opencontainers.image.revision"
|
|
],
|
|
}
|
|
try:
|
|
number = 0
|
|
cases = [
|
|
(repeat, version, pg_cpu)
|
|
for repeat in range(1, 4)
|
|
for pg_cpu in [0.1, 1.0]
|
|
for version in (
|
|
["before", "after"] if repeat % 2 else ["after", "before"]
|
|
)
|
|
]
|
|
for repeat, version, pg_cpu in cases:
|
|
number += 1
|
|
env["API_IMAGE"] = (
|
|
args.before if version == "before" else args.after
|
|
)
|
|
compose(env, "up", "-d", "--no-build", "--wait")
|
|
pg_cid = compose(env, "ps", "-q", "postgres")
|
|
command(["docker", "update", "--cpus", str(pg_cpu), pg_cid], env)
|
|
time.sleep(3)
|
|
assert project.startswith("lab3-compare-")
|
|
compose(
|
|
env,
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-c",
|
|
"TRUNCATE idempotency,messages,members,chats,sessions,users",
|
|
)
|
|
base = f"http://127.0.0.1:{port}"
|
|
with httpx.Client(
|
|
base_url=base, trust_env=False, timeout=15
|
|
) as client:
|
|
user, _, session = actor(client)
|
|
r = client.post(
|
|
"/api/v1/chats",
|
|
headers=auth(session),
|
|
json={
|
|
"title": "Controlled comparison",
|
|
"member_ids": [user["id"]],
|
|
},
|
|
)
|
|
assert r.status_code == 201
|
|
chat = r.json()["id"]
|
|
for i in range(100):
|
|
r = client.post(
|
|
f"/api/v1/chats/{chat}/messages",
|
|
headers={
|
|
**auth(session),
|
|
"Idempotency-Key": uuid4().hex,
|
|
},
|
|
json={"text": f"Seed {i}"},
|
|
)
|
|
assert r.status_code == 201
|
|
asyncio.run(load(base, session["token"], chat, 16, 2))
|
|
counters = cpu_counters(env)
|
|
pg_before = command(
|
|
["docker", "exec", pg_cid, "cat", "/sys/fs/cgroup/cpu.stat"],
|
|
env,
|
|
)
|
|
samples = []
|
|
stop = threading.Event()
|
|
thread = threading.Thread(target=sample, args=(env, stop, samples))
|
|
thread.start()
|
|
try:
|
|
result = asyncio.run(
|
|
load(base, session["token"], chat, 16, 12)
|
|
)
|
|
finally:
|
|
stop.set()
|
|
thread.join()
|
|
result.update(
|
|
version=version,
|
|
repeat=repeat,
|
|
samples=samples,
|
|
pg_cpu_limit=pg_cpu,
|
|
pg_cpu_before=pg_before,
|
|
pg_cpu_after=command(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
pg_cid,
|
|
"cat",
|
|
"/sys/fs/cgroup/cpu.stat",
|
|
],
|
|
env,
|
|
),
|
|
cgroup_before=counters,
|
|
cgroup_after=cpu_counters(env),
|
|
)
|
|
name = f"{number:02}-{version}-pg{pg_cpu}-r{repeat}.json"
|
|
(out / name).write_text(
|
|
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
report["cells"].append(name)
|
|
print(
|
|
f"{name}: {result['successful_rps']:.1f} RPS; {result['errors']} errors",
|
|
flush=True,
|
|
)
|
|
report["complete"] = True
|
|
finally:
|
|
(out / "run.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
compose(env, "down", "--volumes", "--remove-orphans")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|