forked from mtusi2026/mtusi2026lab3
195 lines
8.3 KiB
Python
195 lines
8.3 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
RAW = ROOT / 'evidence' / 'track1' / 'raw'
|
|
PROJECT = 'mtusi-lab3'
|
|
|
|
SERIES = {
|
|
'fixed': {'f1x2.0': (1, '2.0', '512M', 20), 'f2x1.0': (2, '1.0', '256M', 10), 'f4x0.5': (4, '0.5', '128M', 5)},
|
|
'growth': {'g1': (1, '1.0', '256M', 10), 'g2': (2, '1.0', '256M', 10),
|
|
'g4': (4, '1.0', '256M', 10), 'g8': (8, '1.0', '256M', 10)},
|
|
}
|
|
|
|
|
|
def sh(args, env=None, check=True):
|
|
r = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding='utf-8', env=env)
|
|
if check and r.returncode != 0:
|
|
raise RuntimeError(' '.join(args[:4]) + ': ' + r.stderr[-400:])
|
|
return r.stdout
|
|
|
|
|
|
def compose_env(cfg, extra):
|
|
n, cpus, mem, pool = cfg
|
|
env = dict(os.environ, API_REPLICAS=str(n), API_CPUS=cpus, API_MEMORY=mem, DB_MAX_CONNS=str(pool))
|
|
env.update(extra)
|
|
return env
|
|
|
|
|
|
def apply(cfg, extra):
|
|
env = compose_env(cfg, extra)
|
|
sh(['docker', 'compose', 'up', '-d', '--wait', '--force-recreate', '--no-deps', 'api'], env=env)
|
|
sh(['docker', 'compose', 'up', '-d', '--wait', '--no-deps', 'lb'], env=env)
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
seen = set()
|
|
for _ in range(4 * cfg[0] + 4):
|
|
with urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) as r:
|
|
seen.add(r.headers['X-Instance-Id'])
|
|
if len(seen) == cfg[0]:
|
|
return sorted(seen)
|
|
time.sleep(1)
|
|
raise RuntimeError('LB does not see %d replicas' % cfg[0])
|
|
|
|
|
|
def parse_mem(s):
|
|
m = re.match(r'([\d.]+)\s*([KMG]i?B|B)', s.strip())
|
|
if not m:
|
|
return None
|
|
k = {'B': 1 / 2**20, 'KiB': 1 / 1024, 'KB': 1 / 1024, 'MiB': 1, 'MB': 1, 'GiB': 1024, 'GB': 1024}[m.group(2)]
|
|
return float(m.group(1)) * k
|
|
|
|
|
|
class Sampler(threading.Thread):
|
|
def __init__(self):
|
|
super().__init__(daemon=True)
|
|
self.stop = threading.Event()
|
|
self.stats = []
|
|
self.pg = []
|
|
|
|
def run(self):
|
|
while not self.stop.is_set():
|
|
out = sh(['docker', 'stats', '--no-stream', '--format', '{{json .}}'], check=False)
|
|
snap = {}
|
|
for line in out.splitlines():
|
|
try:
|
|
d = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
m = re.match(PROJECT + r'-(\w+?)-\d+$', d['Name'])
|
|
if not m:
|
|
continue
|
|
cpu = float(d['CPUPerc'].rstrip('%') or 0)
|
|
s = snap.setdefault(m.group(1), {'cpu': 0.0, 'mem_mib': 0.0, 'max_cpu': 0.0, 'n': 0})
|
|
s['cpu'] += cpu
|
|
s['max_cpu'] = max(s['max_cpu'], cpu)
|
|
s['mem_mib'] += parse_mem(d['MemUsage'].split('/')[0]) or 0
|
|
s['n'] += 1
|
|
if snap:
|
|
self.stats.append(snap)
|
|
q = ("SELECT coalesce(wait_event_type,'CPU')||':'||coalesce(wait_event,'-'), count(*) "
|
|
"FROM pg_stat_activity WHERE state='active' AND backend_type='client backend' "
|
|
"AND pid<>pg_backend_pid() GROUP BY 1")
|
|
out = sh(['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
|
'-At', '-F', '|', '-c', q], check=False)
|
|
waits = {}
|
|
for line in out.splitlines():
|
|
if '|' in line:
|
|
k, v = line.rsplit('|', 1)
|
|
waits[k] = int(v)
|
|
self.pg.append({'active': sum(waits.values()), 'waits': waits})
|
|
|
|
|
|
K6_CPUS = '4'
|
|
|
|
|
|
def k6(name, vus, duration, run_id, summary=None):
|
|
args = ['docker', 'run', '--rm', '--network', PROJECT + '_app', '--cpus', K6_CPUS,
|
|
'-v', str(ROOT / 'infra' / 'k6') + ':/scripts:ro',
|
|
'-e', 'VUS=%d' % vus, '-e', 'DURATION=' + duration, '-e', 'RUN_ID=' + run_id]
|
|
if summary:
|
|
args += ['-v', str(RAW) + ':/out']
|
|
args += ['grafana/k6:0.54.0', 'run', '--quiet']
|
|
if summary:
|
|
args += ['--summary-export', '/out/' + summary]
|
|
args += ['/scripts/track1.js']
|
|
env = dict(os.environ, MSYS_NO_PATHCONV='1')
|
|
sh(args, env=env)
|
|
|
|
|
|
def aggregate(sampler):
|
|
svc = {}
|
|
for snap in sampler.stats:
|
|
for k, v in snap.items():
|
|
svc.setdefault(k, []).append(v)
|
|
out = {}
|
|
for k, vals in svc.items():
|
|
out[k] = {'cpu_pct_mean': round(sum(v['cpu'] for v in vals) / len(vals), 1),
|
|
'cpu_pct_max_one_container': round(max(v['max_cpu'] for v in vals), 1),
|
|
'mem_mib_mean': round(sum(v['mem_mib'] for v in vals) / len(vals), 1),
|
|
'containers': vals[-1]['n']}
|
|
waits = {}
|
|
for s in sampler.pg:
|
|
for k, v in s['waits'].items():
|
|
waits[k] = waits.get(k, 0) + v
|
|
n = max(1, len(sampler.pg))
|
|
return out, {'samples': len(sampler.pg),
|
|
'active_backends_mean': round(sum(s['active'] for s in sampler.pg) / n, 1),
|
|
'wait_events_mean': {k: round(v / n, 2) for k, v in sorted(waits.items(), key=lambda x: -x[1])}}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--series', choices=list(SERIES), required=True)
|
|
p.add_argument('--only', nargs='*', default=None, help='subset of config names')
|
|
p.add_argument('--repeat', type=int, default=3)
|
|
p.add_argument('--vus', type=int, default=128)
|
|
p.add_argument('--warmup', default='10s')
|
|
p.add_argument('--duration', default='30s')
|
|
p.add_argument('--tag', default='', help='suffix for an experiment variant, e.g. "-v2"')
|
|
p.add_argument('--env', nargs='*', default=[], help='extra compose env KEY=VALUE')
|
|
p.add_argument('--start-repeat', type=int, default=1, help='number of the first repeat (to interleave variants)')
|
|
p.add_argument('--k6-cpus', default='4')
|
|
a = p.parse_args()
|
|
global K6_CPUS
|
|
K6_CPUS = a.k6_cpus
|
|
RAW.mkdir(parents=True, exist_ok=True)
|
|
extra = dict(kv.split('=', 1) for kv in a.env)
|
|
cfgs = {k: v for k, v in SERIES[a.series].items() if not a.only or k in a.only}
|
|
for rep in range(a.start_repeat, a.start_repeat + a.repeat):
|
|
for name, cfg in cfgs.items():
|
|
label = '%s%s-r%d' % (name, a.tag, rep)
|
|
print('==', label, cfg, flush=True)
|
|
replicas = apply(cfg, extra)
|
|
k6(name, a.vus, a.warmup, 'w%s%d' % (name.replace('.', ''), int(time.time())))
|
|
sampler = Sampler()
|
|
sampler.start()
|
|
t0 = time.time()
|
|
run_id = 'm%s%d' % (name.replace('.', ''), int(t0))
|
|
k6(name, a.vus, a.duration, run_id, summary=label + '.k6.json')
|
|
sampler.stop.set()
|
|
sampler.join(timeout=20)
|
|
stats, pg = aggregate(sampler)
|
|
summary_path = RAW / (label + '.k6.json')
|
|
k6sum = json.loads(summary_path.read_text(encoding='utf-8'))
|
|
k6sum.pop('setup_data', None)
|
|
summary_path.write_text(json.dumps(k6sum, indent=2) + '\n', encoding='utf-8')
|
|
k = k6sum['metrics']
|
|
dur = k['http_req_duration']
|
|
rec = {'label': label, 'config': name + a.tag, 'series': a.series, 'repeat': rep,
|
|
'replicas': cfg[0], 'cpus_per_replica': cfg[1], 'mem_per_replica': cfg[2],
|
|
'db_pool_per_replica': cfg[3], 'extra_env': extra, 'k6_cpus': K6_CPUS, 'vus': a.vus, 'duration': a.duration,
|
|
'started_unix': int(t0), 'instances': replicas,
|
|
'rps': round(k['http_reqs']['rate'], 1),
|
|
'error_ratio': round(k['http_req_failed']['value'], 6),
|
|
'latency_ms': {q: round(dur[q], 2) for q in ('p(50)', 'p(95)', 'p(99)', 'max')},
|
|
'docker_stats': stats, 'postgres': pg}
|
|
(RAW / (label + '.json')).write_text(json.dumps(rec, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(' rps=%s err=%s p95=%s api_cpu=%s pg_cpu=%s lb_cpu=%s pg_active=%s' % (
|
|
rec['rps'], rec['error_ratio'], rec['latency_ms']['p(95)'],
|
|
stats.get('api', {}).get('cpu_pct_mean'), stats.get('postgres', {}).get('cpu_pct_mean'),
|
|
stats.get('lb', {}).get('cpu_pct_mean'), pg['active_backends_mean']), flush=True)
|
|
apply((2, '1.0', '256M', 10), {})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|