Files

161 lines
7.4 KiB
Python

#!/usr/bin/env python3
"""Reproducible lab3 track 1. Changes ONLY this Compose project's resources.
Restores the normal two-replica configuration in finally; does not delete volumes.
"""
import argparse
from collections import defaultdict
import json
import os
from pathlib import Path
import random
import subprocess
import threading
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = str(ROOT / 'scripts/compose.sh')
DEST = ROOT / 'evidence/scaling'
def call(*args, env=None):
return subprocess.check_output(args, cwd=ROOT, env=env, text=True).strip()
def compose(*args, env=None):
return call(COMPOSE, *args, env=env)
def inspect(ids):
return json.loads(call('docker', 'inspect', *ids))
def wait_ready(env, replicas, base):
deadline = time.monotonic()+75
while time.monotonic() < deadline:
ids = compose('ps', '-q', 'api', env=env).split()
if len(ids) == replicas:
state = inspect(ids)
if all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in state):
expected = {c['Config']['Hostname'] for c in state}
try:
seen = set()
for _ in range(32):
with urllib.request.urlopen(base+'/health/ready', timeout=2) as response:
seen.add(response.headers.get('X-Instance-Id'))
if seen == expected:
return
except OSError:
pass
time.sleep(.5)
raise RuntimeError('replicas/DNS/LB did not converge')
def dependency_stats(env):
sql = "SELECT json_build_object('xact_commit',xact_commit,'xact_rollback',xact_rollback,'blks_hit',blks_hit,'blks_read',blks_read,'tup_inserted',tup_inserted,'numbackends',numbackends) FROM pg_stat_database WHERE datname='messenger'"
db = json.loads(compose('exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',sql,env=env))
info = compose('exec','-T','valkey','valkey-cli','info','stats',env=env)
cache = {k: int(v.strip()) for line in info.splitlines() if ':' in line for k,v in [line.split(':',1)] if k in ('total_connections_received','total_commands_processed','keyspace_hits','keyspace_misses')}
return {'db':db,'valkey':cache}
class Stats:
def __init__(self, ids):
self.rows = []
self.proc = subprocess.Popen(['docker','stats','--format','{{json .}}','--no-trunc'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
self.thread = threading.Thread(target=self.read, daemon=True)
self.thread.start()
def read(self):
for line in self.proc.stdout:
try:
row = json.loads(line[line.find("{"):])
row['at_unix'] = time.time()
if row.get('Name', '').startswith('mtusi-lab3-'):
self.rows.append(row)
except ValueError:
pass
def close(self):
self.proc.kill()
self.proc.wait(timeout=5)
self.thread.join(timeout=5)
def run_case(case, index, args):
env = dict(os.environ, API_CPUS=str(case['api_cpu']), API_MEMORY=case['api_memory'],
API_REPLICAS=str(case['replicas']), DB_CPUS=str(case.get('db_cpu',.25)),
LB_CPUS=str(case.get('lb_cpu',.25)), DB_POOL_MAX_SIZE=str(case.get('pool',5)),
CACHE_CLIENT_MODE=case.get('cache','shared'),
COMPOSE_FILE=str(ROOT/'compose.yaml')+os.pathsep+str(ROOT/'compose.research.yaml'))
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api='+str(case['replicas']),'api','db','lb'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
wait_ready(env, case['replicas'], args.base_url)
ids = compose('ps','-q','api','db','lb','valkey',env=env).split()
configs = [{'name':c['Name'], 'id':c['Id'], 'image_id':c['Image'],
'cpu_limit':c['HostConfig']['NanoCpus']/1e9, 'memory_bytes':c['HostConfig']['Memory']}
for c in inspect(ids)]
before = dependency_stats(env)
observer = Stats(ids)
try:
output = compose('run','--rm','-T','--no-deps','--name','mtusi-lab3-research-load','load',
'--concurrency',str(case.get('concurrency',args.concurrency)),
'--duration',str(args.duration),'--warmup',str(args.warmup),env=env)
benchmark = json.loads(output)
finally:
observer.close()
after = dependency_stats(env)
cpu = defaultdict(list)
for row in observer.rows:
if benchmark['measure_started_unix'] <= row['at_unix'] <= benchmark['finished_unix']:
cpu[row['Name']].append(float(row['CPUPerc'].rstrip('%')))
report = {'case':case,'repeat':index,'benchmark':benchmark,'containers':configs,
'cpu_percent_mean':{k:round(sum(v)/len(v),3) for k,v in cpu.items()},
'stats_samples':observer.rows,'dependency_before':before,'dependency_after':after}
filename = case['name']+'-'+str(index)+'.json'
(DEST/filename).write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps({'file':filename,'rps':benchmark['successful_rps'],
'p95_ms':benchmark['successful_latency_ms']['p95'],
'errors':benchmark['errors'],'cpu':report['cpu_percent_mean']}), flush=True)
return filename
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url',default=os.getenv('BASE_URL','http://localhost:8083'))
parser.add_argument('--duration',type=float,default=12)
parser.add_argument('--warmup',type=float,default=3)
parser.add_argument('--repeats',type=int,default=3)
parser.add_argument('--concurrency',type=int,default=16)
parser.add_argument('--suite',choices=['scaling','saturation','improvement','all'],default='all')
args=parser.parse_args()
DEST.mkdir(parents=True,exist_ok=True)
cases=[]
if args.suite in ('scaling','all'):
for n in (1,2,4):
cases.append({'name':f'fixed-{n}','replicas':n,'api_cpu':1/n,'api_memory':str(1024//n)+'m','family':'fixed'})
cases.append({'name':f'growing-{n}','replicas':n,'api_cpu':.5,'api_memory':'256m','family':'growing'})
if args.suite in ('saturation','all'):
for concurrency in (1,4,16,64):
cases.append({'name':f'saturation-c{concurrency}','replicas':4,'api_cpu':.5,'api_memory':'256m','concurrency':concurrency,'family':'saturation'})
for db_cpu in (.5,1):
cases.append({'name':f'db-{db_cpu}','replicas':4,'api_cpu':.5,'api_memory':'256m','db_cpu':db_cpu,'family':'db-quota'})
if args.suite in ('improvement','all'):
for cache in ('per_request','shared'):
cases.append({'name':'cache-'+cache,'replicas':2,'api_cpu':.5,'api_memory':'256m','db_cpu':1,'cache':cache,'family':'cache'})
files=[]
try:
for repeat in range(1,args.repeats+1):
ordered=list(cases)
random.Random(20261030+repeat).shuffle(ordered)
for case in ordered:
files.append(run_case(case,repeat,args))
finally:
# Restore normal config from .env, not the last experimental overrides.
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api=2','api','db','lb'],cwd=ROOT,check=True)
(DEST/('manifest-'+args.suite+'.json')).write_text(json.dumps({'suite':args.suite,'parameters':vars(args),'files':files,'seed_base':20261030},indent=2)+'\n')
if __name__=='__main__':
main()