214 lines
12 KiB
Python
214 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab3 scaling pilot. Put in tests/ beside client.py. Python standard library.
|
|
Open-loop arrivals; bounded client concurrency; overload drops are counted.
|
|
Creates synthetic users/chats/messages. Does not change Docker configuration.
|
|
"""
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from client import Client
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
PG_SQL = """SELECT json_build_object(
|
|
'connections',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database()),
|
|
'active',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid()),
|
|
'lock_waits',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock'),
|
|
'waits',(SELECT COALESCE(json_agg(t),'[]'::json) FROM
|
|
(SELECT wait_event_type,wait_event,count(*) AS n FROM pg_stat_activity
|
|
WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active'
|
|
GROUP BY wait_event_type,wait_event) t),
|
|
'db',(SELECT row_to_json(d) FROM
|
|
(SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,
|
|
tup_inserted,deadlocks,temp_bytes FROM pg_stat_database WHERE datname=current_database()) d));"""
|
|
|
|
|
|
def command(args, timeout=20):
|
|
p = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding='utf-8', errors='replace', timeout=timeout)
|
|
if p.returncode:
|
|
raise RuntimeError('Command failed: ' + ' '.join(args[:4]))
|
|
return p.stdout.strip()
|
|
|
|
|
|
def percentile(values, q):
|
|
if not values:
|
|
return None
|
|
values = sorted(values)
|
|
return round(values[max(0, math.ceil(len(values)*q)-1)], 2)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--label', required=True)
|
|
p.add_argument('--rates', default='10,25,50,100,200')
|
|
p.add_argument('--seconds', type=int, default=30)
|
|
p.add_argument('--warmup', type=int, default=15)
|
|
p.add_argument('--workers', type=int, default=128)
|
|
p.add_argument('--timeout', type=float, default=3)
|
|
args = p.parse_args()
|
|
rates = [float(x) for x in args.rates.split(',')]
|
|
if any(r <= 0 for r in rates) or args.seconds < 5 or args.workers < 1 or args.warmup < 0:
|
|
p.error('Positive rates/workers, duration >=5s, warmup >=0 required')
|
|
label = ''.join(x for x in args.label if x.isalnum() or x in '-_')
|
|
output = ROOT / 'results' / ('scale-' + label + '-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
|
|
output.mkdir(parents=True)
|
|
c = Client(timeout=args.timeout)
|
|
alice = c.user('scale')
|
|
metadata = {'args': vars(args), 'base_url': c.base,
|
|
'profile': '80% GET chat history limit=20, 20% POST unique message; one session; deterministic schedule',
|
|
'limits': [], 'summary': [], 'utc_started': datetime.now(timezone.utc).isoformat()}
|
|
ids = command(['docker','compose','ps','-q']).splitlines()
|
|
for cid in ids:
|
|
raw = json.loads(command(['docker','inspect',cid]))[0]
|
|
if raw['Config']['Labels'].get('com.docker.compose.service') == 'migrate':
|
|
continue
|
|
env = dict(x.split('=',1) for x in raw['Config'].get('Env',[]) if '=' in x)
|
|
metadata['limits'].append({'id':cid,'name':raw['Name'],
|
|
'service':raw['Config']['Labels'].get('com.docker.compose.service'),
|
|
'image':raw['Image'],'cpu':raw['HostConfig'].get('NanoCpus',0)/1e9,
|
|
'memory_bytes':raw['HostConfig'].get('Memory'),
|
|
'db_pool_size':env.get('DB_POOL_SIZE')})
|
|
del raw, env
|
|
metadata['docker_resources'] = json.loads(command(['docker','info','--format',
|
|
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}']))
|
|
api_names = {x['id'][:12] for x in metadata['limits'] if x['service']=='api'}
|
|
seen = set()
|
|
for _ in range(40):
|
|
seen.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
|
if not api_names <= seen:
|
|
raise RuntimeError('Not all running API containers were observed through LB')
|
|
metadata['observed_instances'] = sorted(seen)
|
|
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
|
|
|
|
stop_sample = threading.Event()
|
|
phase = {'name': 'setup', 'rate': 0}
|
|
origin = time.monotonic()
|
|
def sampler():
|
|
with (output/'resources.jsonl').open('w',encoding='utf-8') as f:
|
|
while not stop_sample.is_set():
|
|
record = {'seconds':time.monotonic()-origin, **phase.copy()}
|
|
try:
|
|
stats = command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=10)
|
|
record['docker'] = [json.loads(line) for line in stats.splitlines() if line.strip()]
|
|
except Exception as exc:
|
|
record['docker_error'] = type(exc).__name__
|
|
try:
|
|
# Variables are expanded in the container, never printed or persisted.
|
|
pg = command(['docker','compose','exec','-T','postgres','sh','-c',
|
|
'exec psql -X -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"',
|
|
'sh',PG_SQL],timeout=10)
|
|
record['postgres'] = json.loads(pg)
|
|
except Exception as exc:
|
|
record['postgres_error'] = type(exc).__name__
|
|
record['sample_end_seconds'] = time.monotonic()-origin
|
|
f.write(json.dumps(record)+'\n'); f.flush()
|
|
stop_sample.wait(2)
|
|
monitor = threading.Thread(target=sampler,daemon=True)
|
|
monitor.start()
|
|
local = threading.local()
|
|
lock = threading.Lock()
|
|
slots = threading.BoundedSemaphore(args.workers)
|
|
fields = ['stage','rate','index','method','scheduled','started','finished','latency_ms',
|
|
'schedule_lag_ms','status','instance','ok','error']
|
|
try:
|
|
with (output/'requests.csv').open('w',newline='',encoding='utf-8') as file:
|
|
writer = csv.DictWriter(file,fieldnames=fields);writer.writeheader()
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
for rate in rates:
|
|
# Each rate starts with a fresh chat and identical 20-message seed.
|
|
chat = c.call('POST','/api/v1/chats',201,actor=alice,
|
|
body={'title':'Scaling benchmark','member_ids':[alice['id']]},schema='Chat')['json']
|
|
for i in range(20):
|
|
c.message(alice,chat,'seed-'+str(i))
|
|
path = '/api/v1/chats/'+chat['id']+'/messages'
|
|
for stage, duration in [('warmup',args.warmup),('measure',args.seconds)]:
|
|
if not duration:continue
|
|
phase.update(name=stage,rate=rate)
|
|
rows = []
|
|
start = time.monotonic()
|
|
def save(row):
|
|
with lock:
|
|
rows.append(row);writer.writerow(row)
|
|
def request(index, due, stage=stage, rate=rate, path=path, rows=rows):
|
|
started = time.monotonic()
|
|
row = dict(stage=stage,rate=rate,index=index,method='POST' if index%5==0 else 'GET',
|
|
scheduled=due-origin,started=started-origin,finished='',latency_ms='',
|
|
schedule_lag_ms=round((started-due)*1000,3),status=0,instance='',ok=False,error='')
|
|
try:
|
|
if not hasattr(local,'client'):local.client=Client(timeout=args.timeout)
|
|
method=row['method']
|
|
# Benchmark does NOT retry: every original attempt stays visible.
|
|
key=uuid.uuid4().hex
|
|
r=local.client.call(method,path if method=='POST' else path+'?limit=20',
|
|
expected=[200,201,400,401,403,404,409,429,500,502,503,504],actor=alice,
|
|
body={'text':'scaling-'+key} if method=='POST' else None,
|
|
headers={'Idempotency-Key':key} if method=='POST' else None,
|
|
schema='Message' if method=='POST' else 'MessagePage')
|
|
row.update(status=r['status'],instance=r['headers'].get('X-Instance-Id',''),
|
|
ok=r['status']==(201 if method=='POST' else 200))
|
|
if not row['ok']:row['error']='HTTP '+str(r['status'])
|
|
except Exception as exc:
|
|
row['error']=str(exc) if isinstance(exc,AssertionError) else type(exc).__name__
|
|
finally:
|
|
finish=time.monotonic()
|
|
row.update(finished=finish-origin,latency_ms=round((finish-started)*1000,3))
|
|
save(row);slots.release()
|
|
futures=[]
|
|
for i in range(math.ceil(rate*duration)):
|
|
due=start+i/rate
|
|
delay=due-time.monotonic()
|
|
if delay>0:time.sleep(delay)
|
|
now=time.monotonic()
|
|
# Do not create catch-up bursts or silently queue unlimited work.
|
|
if now-due>.1 or not slots.acquire(blocking=False):
|
|
save(dict(stage=stage,rate=rate,index=i,method='POST' if i%5==0 else 'GET',
|
|
scheduled=due-origin,started='',finished=now-origin,latency_ms='',
|
|
schedule_lag_ms=round((now-due)*1000,3),status=0,instance='',ok=False,
|
|
error='generator_drop'))
|
|
else:
|
|
futures.append(pool.submit(request,i,due))
|
|
remainder=start+duration-time.monotonic()
|
|
if remainder>0:time.sleep(remainder)
|
|
for f in futures:f.result()
|
|
ended=time.monotonic()
|
|
file.flush()
|
|
if stage=='measure':
|
|
good=[r for r in rows if r['ok']]
|
|
latency=[r['latency_ms'] for r in good]
|
|
drops=sum(r['error']=='generator_drop' for r in rows)
|
|
summary={'target_rps':rate,'scheduled_seconds':duration,
|
|
'elapsed_with_drain_seconds':round(ended-start,3),'scheduled_requests':len(rows),
|
|
'sent':len(rows)-drops,'generator_drops':drops,'successes':len(good),
|
|
'errors':len(rows)-len(good)-drops,
|
|
'success_rps_with_drain':round(len(good)/(ended-start),3),
|
|
'sent_rps':round((len(rows)-drops)/duration,3),
|
|
'failure_pct_including_drops':round(100*(len(rows)-len(good))/len(rows),3),
|
|
'p50_ms_success':percentile(latency,.5),'p95_ms_success':percentile(latency,.95),
|
|
'p99_ms_success':percentile(latency,.99),
|
|
'p95_schedule_lag_ms':percentile([r['schedule_lag_ms'] for r in rows],.95),
|
|
'instances':sorted({r['instance'] for r in good})}
|
|
metadata['summary'].append(summary)
|
|
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
|
|
print(json.dumps(summary),flush=True)
|
|
phase.update(name='cooldown',rate=rate)
|
|
time.sleep(5)
|
|
finally:
|
|
stop_sample.set();monitor.join(timeout=25)
|
|
print('Results:',output,flush=True)
|
|
print('DONE: send summary.json and resources.jsonl for saturation analysis')
|
|
|
|
|
|
if __name__=='__main__':
|
|
try:main()
|
|
except Exception as exc:
|
|
print('FAIL:',str(exc));raise SystemExit(1)
|