forked from mtusi2026/mtusi2026lab3
66 lines
3.6 KiB
Python
Executable File
66 lines
3.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Full lab3 scaling matrix; a new chat and bounded generator for every run."""
|
|
import argparse,datetime,json,os,random,subprocess,threading,time
|
|
from pathlib import Path
|
|
from metrics import ROOT,containers,snapshot
|
|
|
|
def run(args,env=None):
|
|
p=subprocess.run(args,cwd=ROOT,env=env,text=True,capture_output=True,timeout=180)
|
|
if p.returncode:raise RuntimeError(' '.join(args[:3])+': '+p.stderr[-500:])
|
|
return p.stdout.strip()
|
|
def configure(replicas,cpus,memory):
|
|
env=dict(os.environ,API_IMAGE='mtusi-messenger:lab3',API_REPLICAS=str(replicas),API_CPUS=str(cpus),API_MEMORY=str(memory)+'m',POSTGRES_MAX_CONNECTIONS=str(16//replicas))
|
|
run(['./scripts/compose.sh','up','-d','--no-deps','--wait','--wait-timeout','90','--scale',f'api={replicas}','api'],env)
|
|
# Explicitly wait for Docker DNS refresh and LB health checks after an address change.
|
|
time.sleep(4)
|
|
return env
|
|
|
|
def measure(base,label,rate,duration,warmup,seed):
|
|
dest=ROOT/'evidence/research'/f'{label}.json'
|
|
if dest.exists():return
|
|
# Never begin setup while backends are still recovering from the previous overload.
|
|
import urllib.request
|
|
deadline=time.monotonic()+45
|
|
consecutive=0
|
|
while consecutive<3:
|
|
try:
|
|
with urllib.request.urlopen(base+'/health/ready',timeout=2) as r:consecutive=consecutive+1 if r.status==200 else 0
|
|
except Exception:consecutive=0
|
|
if time.monotonic()>deadline:raise RuntimeError('LB did not recover after overload')
|
|
time.sleep(.5)
|
|
rows=containers();stop=threading.Event();metrics=dest.with_suffix('.metrics.jsonl')
|
|
def collect():
|
|
with metrics.open('w') as f:
|
|
while not stop.is_set():
|
|
f.write(json.dumps(snapshot(rows))+'\n');f.flush();stop.wait(3)
|
|
thread=threading.Thread(target=collect);thread.start()
|
|
print(label,flush=True)
|
|
try:
|
|
p=subprocess.run([str(ROOT/'bin/benchmark'),'-base-url',base,'-rate',str(rate),'-duration',duration,'-warmup',warmup,'-seed',str(seed),'-label',label,'-output',str(dest)],cwd=ROOT,stdout=subprocess.DEVNULL,timeout=180)
|
|
if p.returncode:raise RuntimeError(f'benchmark failed: {label}')
|
|
finally:stop.set();thread.join()
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--base-url',default='http://localhost:18080');p.add_argument('--comparison-only',action='store_true');a=p.parse_args()
|
|
folder=ROOT/'evidence/research';folder.mkdir(exist_ok=True)
|
|
meta={'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'commit':run(['git','rev-parse','HEAD']),'docker':run(['docker','info','--format','{{.NCPU}} CPUs / {{.MemTotal}} bytes']),'matrix':{'replicas':[1,2,4],'rates':[100,300,900,2700,8100],'repeats':3,'warmup_seconds':5,'duration_seconds':30,'seed_messages':1000,'total_pool_connections':16},'note':'Running lab2 is kept intact and idle. Metric sampling is included in every run. API, LB and DB share one Docker Desktop VM.'}
|
|
(folder/'method.json').write_text(json.dumps(meta,indent=2)+'\n')
|
|
try:
|
|
configure(2,1,512)
|
|
for rate in (5,20,80):
|
|
for repeat in range(1,4):measure(a.base_url,f'after-2-r{rate}-n{repeat}',rate,'15s','3s',1000)
|
|
if a.comparison_only:return
|
|
# Baseline comparisons use the exact preserved generator binary.
|
|
# Rebuild the corrected due-time scheduler for the scaling matrix.
|
|
run(['make','build'])
|
|
for series in ('fixed','growing'):
|
|
for n in (1,2,4):
|
|
cpus,memory=(2/n,1024//n) if series=='fixed' else (1,512)
|
|
configure(n,cpus,memory)
|
|
cases=[(rate,rep) for rate in (100,300,900,2700,8100) for rep in (1,2,3)]
|
|
random.Random(20261106).shuffle(cases)
|
|
for rate,rep in cases:measure(a.base_url,f'{series}-{n}-r{rate}-n{rep}',rate,'30s','5s',1000)
|
|
finally:configure(2,1,512)
|
|
print('Research matrix complete; restored 2 API replicas.',flush=True)
|
|
if __name__=='__main__':main()
|