#!/usr/bin/env python3 """Reproducible lab3 track 1. Changes ONLY this Compose project's resources. Restores the normal two-replica configuration in finally; does not delete volumes. """ import argparse from collections import defaultdict import json import os from pathlib import Path import random import subprocess import threading import time import urllib.request ROOT = Path(__file__).resolve().parents[1] COMPOSE = str(ROOT / 'scripts/compose.sh') DEST = ROOT / 'evidence/scaling' def call(*args, env=None): return subprocess.check_output(args, cwd=ROOT, env=env, text=True).strip() def compose(*args, env=None): return call(COMPOSE, *args, env=env) def inspect(ids): return json.loads(call('docker', 'inspect', *ids)) def wait_ready(env, replicas, base): deadline = time.monotonic()+75 while time.monotonic() < deadline: ids = compose('ps', '-q', 'api', env=env).split() if len(ids) == replicas: state = inspect(ids) if all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in state): expected = {c['Config']['Hostname'] for c in state} try: seen = set() for _ in range(32): with urllib.request.urlopen(base+'/health/ready', timeout=2) as response: seen.add(response.headers.get('X-Instance-Id')) if seen == expected: return except OSError: pass time.sleep(.5) raise RuntimeError('replicas/DNS/LB did not converge') def dependency_stats(env): sql = "SELECT json_build_object('xact_commit',xact_commit,'xact_rollback',xact_rollback,'blks_hit',blks_hit,'blks_read',blks_read,'tup_inserted',tup_inserted,'numbackends',numbackends) FROM pg_stat_database WHERE datname='messenger'" db = json.loads(compose('exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',sql,env=env)) info = compose('exec','-T','valkey','valkey-cli','info','stats',env=env) cache = {k: int(v.strip()) for line in info.splitlines() if ':' in line for k,v in [line.split(':',1)] if k in ('total_connections_received','total_commands_processed','keyspace_hits','keyspace_misses')} return {'db':db,'valkey':cache} class Stats: def __init__(self, ids): self.rows = [] self.proc = subprocess.Popen(['docker','stats','--format','{{json .}}','--no-trunc'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) self.thread = threading.Thread(target=self.read, daemon=True) self.thread.start() def read(self): for line in self.proc.stdout: try: row = json.loads(line[line.find("{"):]) row['at_unix'] = time.time() if row.get('Name', '').startswith('mtusi-lab3-'): self.rows.append(row) except ValueError: pass def close(self): self.proc.kill() self.proc.wait(timeout=5) self.thread.join(timeout=5) def run_case(case, index, args): env = dict(os.environ, API_CPUS=str(case['api_cpu']), API_MEMORY=case['api_memory'], API_REPLICAS=str(case['replicas']), DB_CPUS=str(case.get('db_cpu',.25)), LB_CPUS=str(case.get('lb_cpu',.25)), DB_POOL_MAX_SIZE=str(case.get('pool',5)), CACHE_CLIENT_MODE=case.get('cache','shared'), COMPOSE_FILE=str(ROOT/'compose.yaml')+os.pathsep+str(ROOT/'compose.research.yaml')) subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api='+str(case['replicas']),'api','db','lb'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL) wait_ready(env, case['replicas'], args.base_url) ids = compose('ps','-q','api','db','lb','valkey',env=env).split() configs = [{'name':c['Name'], 'id':c['Id'], 'image_id':c['Image'], 'cpu_limit':c['HostConfig']['NanoCpus']/1e9, 'memory_bytes':c['HostConfig']['Memory']} for c in inspect(ids)] before = dependency_stats(env) observer = Stats(ids) try: output = compose('run','--rm','-T','--no-deps','--name','mtusi-lab3-research-load','load', '--concurrency',str(case.get('concurrency',args.concurrency)), '--duration',str(args.duration),'--warmup',str(args.warmup),env=env) benchmark = json.loads(output) finally: observer.close() after = dependency_stats(env) cpu = defaultdict(list) for row in observer.rows: if benchmark['measure_started_unix'] <= row['at_unix'] <= benchmark['finished_unix']: cpu[row['Name']].append(float(row['CPUPerc'].rstrip('%'))) report = {'case':case,'repeat':index,'benchmark':benchmark,'containers':configs, 'cpu_percent_mean':{k:round(sum(v)/len(v),3) for k,v in cpu.items()}, 'stats_samples':observer.rows,'dependency_before':before,'dependency_after':after} filename = case['name']+'-'+str(index)+'.json' (DEST/filename).write_text(json.dumps(report, indent=2)+'\n') print(json.dumps({'file':filename,'rps':benchmark['successful_rps'], 'p95_ms':benchmark['successful_latency_ms']['p95'], 'errors':benchmark['errors'],'cpu':report['cpu_percent_mean']}), flush=True) return filename def main(): parser=argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url',default=os.getenv('BASE_URL','http://localhost:8083')) parser.add_argument('--duration',type=float,default=12) parser.add_argument('--warmup',type=float,default=3) parser.add_argument('--repeats',type=int,default=3) parser.add_argument('--concurrency',type=int,default=16) parser.add_argument('--suite',choices=['scaling','saturation','improvement','all'],default='all') args=parser.parse_args() DEST.mkdir(parents=True,exist_ok=True) cases=[] if args.suite in ('scaling','all'): for n in (1,2,4): cases.append({'name':f'fixed-{n}','replicas':n,'api_cpu':1/n,'api_memory':str(1024//n)+'m','family':'fixed'}) cases.append({'name':f'growing-{n}','replicas':n,'api_cpu':.5,'api_memory':'256m','family':'growing'}) if args.suite in ('saturation','all'): for concurrency in (1,4,16,64): cases.append({'name':f'saturation-c{concurrency}','replicas':4,'api_cpu':.5,'api_memory':'256m','concurrency':concurrency,'family':'saturation'}) for db_cpu in (.5,1): cases.append({'name':f'db-{db_cpu}','replicas':4,'api_cpu':.5,'api_memory':'256m','db_cpu':db_cpu,'family':'db-quota'}) if args.suite in ('improvement','all'): for cache in ('per_request','shared'): cases.append({'name':'cache-'+cache,'replicas':2,'api_cpu':.5,'api_memory':'256m','db_cpu':1,'cache':cache,'family':'cache'}) files=[] try: for repeat in range(1,args.repeats+1): ordered=list(cases) random.Random(20261030+repeat).shuffle(ordered) for case in ordered: files.append(run_case(case,repeat,args)) finally: # Restore normal config from .env, not the last experimental overrides. subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api=2','api','db','lb'],cwd=ROOT,check=True) (DEST/('manifest-'+args.suite+'.json')).write_text(json.dumps({'suite':args.suite,'parameters':vars(args),'files':files,'seed_base':20261030},indent=2)+'\n') if __name__=='__main__': main()