forked from mtusi2026/mtusi2026lab3
161 lines
7.4 KiB
Python
161 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Reproducible lab3 track 1. Changes ONLY this Compose project's resources.
|
|
|
|
Restores the normal two-replica configuration in finally; does not delete volumes.
|
|
"""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
COMPOSE = str(ROOT / 'scripts/compose.sh')
|
|
DEST = ROOT / 'evidence/scaling'
|
|
|
|
|
|
def call(*args, env=None):
|
|
return subprocess.check_output(args, cwd=ROOT, env=env, text=True).strip()
|
|
|
|
|
|
def compose(*args, env=None):
|
|
return call(COMPOSE, *args, env=env)
|
|
|
|
|
|
def inspect(ids):
|
|
return json.loads(call('docker', 'inspect', *ids))
|
|
|
|
|
|
def wait_ready(env, replicas, base):
|
|
deadline = time.monotonic()+75
|
|
while time.monotonic() < deadline:
|
|
ids = compose('ps', '-q', 'api', env=env).split()
|
|
if len(ids) == replicas:
|
|
state = inspect(ids)
|
|
if all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in state):
|
|
expected = {c['Config']['Hostname'] for c in state}
|
|
try:
|
|
seen = set()
|
|
for _ in range(32):
|
|
with urllib.request.urlopen(base+'/health/ready', timeout=2) as response:
|
|
seen.add(response.headers.get('X-Instance-Id'))
|
|
if seen == expected:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(.5)
|
|
raise RuntimeError('replicas/DNS/LB did not converge')
|
|
|
|
|
|
def dependency_stats(env):
|
|
sql = "SELECT json_build_object('xact_commit',xact_commit,'xact_rollback',xact_rollback,'blks_hit',blks_hit,'blks_read',blks_read,'tup_inserted',tup_inserted,'numbackends',numbackends) FROM pg_stat_database WHERE datname='messenger'"
|
|
db = json.loads(compose('exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',sql,env=env))
|
|
info = compose('exec','-T','valkey','valkey-cli','info','stats',env=env)
|
|
cache = {k: int(v.strip()) for line in info.splitlines() if ':' in line for k,v in [line.split(':',1)] if k in ('total_connections_received','total_commands_processed','keyspace_hits','keyspace_misses')}
|
|
return {'db':db,'valkey':cache}
|
|
|
|
|
|
class Stats:
|
|
def __init__(self, ids):
|
|
self.rows = []
|
|
self.proc = subprocess.Popen(['docker','stats','--format','{{json .}}','--no-trunc'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
|
self.thread = threading.Thread(target=self.read, daemon=True)
|
|
self.thread.start()
|
|
|
|
def read(self):
|
|
for line in self.proc.stdout:
|
|
try:
|
|
row = json.loads(line[line.find("{"):])
|
|
row['at_unix'] = time.time()
|
|
if row.get('Name', '').startswith('mtusi-lab3-'):
|
|
self.rows.append(row)
|
|
except ValueError:
|
|
pass
|
|
|
|
def close(self):
|
|
self.proc.kill()
|
|
self.proc.wait(timeout=5)
|
|
self.thread.join(timeout=5)
|
|
|
|
|
|
def run_case(case, index, args):
|
|
env = dict(os.environ, API_CPUS=str(case['api_cpu']), API_MEMORY=case['api_memory'],
|
|
API_REPLICAS=str(case['replicas']), DB_CPUS=str(case.get('db_cpu',.25)),
|
|
LB_CPUS=str(case.get('lb_cpu',.25)), DB_POOL_MAX_SIZE=str(case.get('pool',5)),
|
|
CACHE_CLIENT_MODE=case.get('cache','shared'),
|
|
COMPOSE_FILE=str(ROOT/'compose.yaml')+os.pathsep+str(ROOT/'compose.research.yaml'))
|
|
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api='+str(case['replicas']),'api','db','lb'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
|
|
wait_ready(env, case['replicas'], args.base_url)
|
|
ids = compose('ps','-q','api','db','lb','valkey',env=env).split()
|
|
configs = [{'name':c['Name'], 'id':c['Id'], 'image_id':c['Image'],
|
|
'cpu_limit':c['HostConfig']['NanoCpus']/1e9, 'memory_bytes':c['HostConfig']['Memory']}
|
|
for c in inspect(ids)]
|
|
before = dependency_stats(env)
|
|
observer = Stats(ids)
|
|
try:
|
|
output = compose('run','--rm','-T','--no-deps','--name','mtusi-lab3-research-load','load',
|
|
'--concurrency',str(case.get('concurrency',args.concurrency)),
|
|
'--duration',str(args.duration),'--warmup',str(args.warmup),env=env)
|
|
benchmark = json.loads(output)
|
|
finally:
|
|
observer.close()
|
|
after = dependency_stats(env)
|
|
cpu = defaultdict(list)
|
|
for row in observer.rows:
|
|
if benchmark['measure_started_unix'] <= row['at_unix'] <= benchmark['finished_unix']:
|
|
cpu[row['Name']].append(float(row['CPUPerc'].rstrip('%')))
|
|
report = {'case':case,'repeat':index,'benchmark':benchmark,'containers':configs,
|
|
'cpu_percent_mean':{k:round(sum(v)/len(v),3) for k,v in cpu.items()},
|
|
'stats_samples':observer.rows,'dependency_before':before,'dependency_after':after}
|
|
filename = case['name']+'-'+str(index)+'.json'
|
|
(DEST/filename).write_text(json.dumps(report, indent=2)+'\n')
|
|
print(json.dumps({'file':filename,'rps':benchmark['successful_rps'],
|
|
'p95_ms':benchmark['successful_latency_ms']['p95'],
|
|
'errors':benchmark['errors'],'cpu':report['cpu_percent_mean']}), flush=True)
|
|
return filename
|
|
|
|
|
|
def main():
|
|
parser=argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url',default=os.getenv('BASE_URL','http://localhost:8083'))
|
|
parser.add_argument('--duration',type=float,default=12)
|
|
parser.add_argument('--warmup',type=float,default=3)
|
|
parser.add_argument('--repeats',type=int,default=3)
|
|
parser.add_argument('--concurrency',type=int,default=16)
|
|
parser.add_argument('--suite',choices=['scaling','saturation','improvement','all'],default='all')
|
|
args=parser.parse_args()
|
|
DEST.mkdir(parents=True,exist_ok=True)
|
|
cases=[]
|
|
if args.suite in ('scaling','all'):
|
|
for n in (1,2,4):
|
|
cases.append({'name':f'fixed-{n}','replicas':n,'api_cpu':1/n,'api_memory':str(1024//n)+'m','family':'fixed'})
|
|
cases.append({'name':f'growing-{n}','replicas':n,'api_cpu':.5,'api_memory':'256m','family':'growing'})
|
|
if args.suite in ('saturation','all'):
|
|
for concurrency in (1,4,16,64):
|
|
cases.append({'name':f'saturation-c{concurrency}','replicas':4,'api_cpu':.5,'api_memory':'256m','concurrency':concurrency,'family':'saturation'})
|
|
for db_cpu in (.5,1):
|
|
cases.append({'name':f'db-{db_cpu}','replicas':4,'api_cpu':.5,'api_memory':'256m','db_cpu':db_cpu,'family':'db-quota'})
|
|
if args.suite in ('improvement','all'):
|
|
for cache in ('per_request','shared'):
|
|
cases.append({'name':'cache-'+cache,'replicas':2,'api_cpu':.5,'api_memory':'256m','db_cpu':1,'cache':cache,'family':'cache'})
|
|
files=[]
|
|
try:
|
|
for repeat in range(1,args.repeats+1):
|
|
ordered=list(cases)
|
|
random.Random(20261030+repeat).shuffle(ordered)
|
|
for case in ordered:
|
|
files.append(run_case(case,repeat,args))
|
|
finally:
|
|
# Restore normal config from .env, not the last experimental overrides.
|
|
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api=2','api','db','lb'],cwd=ROOT,check=True)
|
|
(DEST/('manifest-'+args.suite+'.json')).write_text(json.dumps({'suite':args.suite,'parameters':vars(args),'files':files,'seed_base':20261030},indent=2)+'\n')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|