Files
mtusi2026lab3/tests/scale_bench.py
2026-10-26 21:32:36 +03:00

214 lines
12 KiB
Python

#!/usr/bin/env python3
"""Lab3 scaling pilot. Put in tests/ beside client.py. Python standard library.
Open-loop arrivals; bounded client concurrency; overload drops are counted.
Creates synthetic users/chats/messages. Does not change Docker configuration.
"""
import argparse
import csv
import json
import math
import os
import subprocess
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from pathlib import Path
from client import Client
ROOT = Path(__file__).resolve().parents[1]
PG_SQL = """SELECT json_build_object(
'connections',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database()),
'active',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND state='active' AND pid<>pg_backend_pid()),
'lock_waits',(SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event_type='Lock'),
'waits',(SELECT COALESCE(json_agg(t),'[]'::json) FROM
(SELECT wait_event_type,wait_event,count(*) AS n FROM pg_stat_activity
WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active'
GROUP BY wait_event_type,wait_event) t),
'db',(SELECT row_to_json(d) FROM
(SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_returned,tup_fetched,
tup_inserted,deadlocks,temp_bytes FROM pg_stat_database WHERE datname=current_database()) d));"""
def command(args, timeout=20):
p = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace', timeout=timeout)
if p.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args[:4]))
return p.stdout.strip()
def percentile(values, q):
if not values:
return None
values = sorted(values)
return round(values[max(0, math.ceil(len(values)*q)-1)], 2)
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--label', required=True)
p.add_argument('--rates', default='10,25,50,100,200')
p.add_argument('--seconds', type=int, default=30)
p.add_argument('--warmup', type=int, default=15)
p.add_argument('--workers', type=int, default=128)
p.add_argument('--timeout', type=float, default=3)
args = p.parse_args()
rates = [float(x) for x in args.rates.split(',')]
if any(r <= 0 for r in rates) or args.seconds < 5 or args.workers < 1 or args.warmup < 0:
p.error('Positive rates/workers, duration >=5s, warmup >=0 required')
label = ''.join(x for x in args.label if x.isalnum() or x in '-_')
output = ROOT / 'results' / ('scale-' + label + '-' + datetime.now(timezone.utc).strftime('%Y%m%d-%H%M%S'))
output.mkdir(parents=True)
c = Client(timeout=args.timeout)
alice = c.user('scale')
metadata = {'args': vars(args), 'base_url': c.base,
'profile': '80% GET chat history limit=20, 20% POST unique message; one session; deterministic schedule',
'limits': [], 'summary': [], 'utc_started': datetime.now(timezone.utc).isoformat()}
ids = command(['docker','compose','ps','-q']).splitlines()
for cid in ids:
raw = json.loads(command(['docker','inspect',cid]))[0]
if raw['Config']['Labels'].get('com.docker.compose.service') == 'migrate':
continue
env = dict(x.split('=',1) for x in raw['Config'].get('Env',[]) if '=' in x)
metadata['limits'].append({'id':cid,'name':raw['Name'],
'service':raw['Config']['Labels'].get('com.docker.compose.service'),
'image':raw['Image'],'cpu':raw['HostConfig'].get('NanoCpus',0)/1e9,
'memory_bytes':raw['HostConfig'].get('Memory'),
'db_pool_size':env.get('DB_POOL_SIZE')})
del raw, env
metadata['docker_resources'] = json.loads(command(['docker','info','--format',
'{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}']))
api_names = {x['id'][:12] for x in metadata['limits'] if x['service']=='api'}
seen = set()
for _ in range(40):
seen.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
if not api_names <= seen:
raise RuntimeError('Not all running API containers were observed through LB')
metadata['observed_instances'] = sorted(seen)
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
stop_sample = threading.Event()
phase = {'name': 'setup', 'rate': 0}
origin = time.monotonic()
def sampler():
with (output/'resources.jsonl').open('w',encoding='utf-8') as f:
while not stop_sample.is_set():
record = {'seconds':time.monotonic()-origin, **phase.copy()}
try:
stats = command(['docker','stats','--no-stream','--format','{{json .}}',*ids],timeout=10)
record['docker'] = [json.loads(line) for line in stats.splitlines() if line.strip()]
except Exception as exc:
record['docker_error'] = type(exc).__name__
try:
# Variables are expanded in the container, never printed or persisted.
pg = command(['docker','compose','exec','-T','postgres','sh','-c',
'exec psql -X -A -t -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"',
'sh',PG_SQL],timeout=10)
record['postgres'] = json.loads(pg)
except Exception as exc:
record['postgres_error'] = type(exc).__name__
record['sample_end_seconds'] = time.monotonic()-origin
f.write(json.dumps(record)+'\n'); f.flush()
stop_sample.wait(2)
monitor = threading.Thread(target=sampler,daemon=True)
monitor.start()
local = threading.local()
lock = threading.Lock()
slots = threading.BoundedSemaphore(args.workers)
fields = ['stage','rate','index','method','scheduled','started','finished','latency_ms',
'schedule_lag_ms','status','instance','ok','error']
try:
with (output/'requests.csv').open('w',newline='',encoding='utf-8') as file:
writer = csv.DictWriter(file,fieldnames=fields);writer.writeheader()
with ThreadPoolExecutor(max_workers=args.workers) as pool:
for rate in rates:
# Each rate starts with a fresh chat and identical 20-message seed.
chat = c.call('POST','/api/v1/chats',201,actor=alice,
body={'title':'Scaling benchmark','member_ids':[alice['id']]},schema='Chat')['json']
for i in range(20):
c.message(alice,chat,'seed-'+str(i))
path = '/api/v1/chats/'+chat['id']+'/messages'
for stage, duration in [('warmup',args.warmup),('measure',args.seconds)]:
if not duration:continue
phase.update(name=stage,rate=rate)
rows = []
start = time.monotonic()
def save(row):
with lock:
rows.append(row);writer.writerow(row)
def request(index, due, stage=stage, rate=rate, path=path, rows=rows):
started = time.monotonic()
row = dict(stage=stage,rate=rate,index=index,method='POST' if index%5==0 else 'GET',
scheduled=due-origin,started=started-origin,finished='',latency_ms='',
schedule_lag_ms=round((started-due)*1000,3),status=0,instance='',ok=False,error='')
try:
if not hasattr(local,'client'):local.client=Client(timeout=args.timeout)
method=row['method']
# Benchmark does NOT retry: every original attempt stays visible.
key=uuid.uuid4().hex
r=local.client.call(method,path if method=='POST' else path+'?limit=20',
expected=[200,201,400,401,403,404,409,429,500,502,503,504],actor=alice,
body={'text':'scaling-'+key} if method=='POST' else None,
headers={'Idempotency-Key':key} if method=='POST' else None,
schema='Message' if method=='POST' else 'MessagePage')
row.update(status=r['status'],instance=r['headers'].get('X-Instance-Id',''),
ok=r['status']==(201 if method=='POST' else 200))
if not row['ok']:row['error']='HTTP '+str(r['status'])
except Exception as exc:
row['error']=str(exc) if isinstance(exc,AssertionError) else type(exc).__name__
finally:
finish=time.monotonic()
row.update(finished=finish-origin,latency_ms=round((finish-started)*1000,3))
save(row);slots.release()
futures=[]
for i in range(math.ceil(rate*duration)):
due=start+i/rate
delay=due-time.monotonic()
if delay>0:time.sleep(delay)
now=time.monotonic()
# Do not create catch-up bursts or silently queue unlimited work.
if now-due>.1 or not slots.acquire(blocking=False):
save(dict(stage=stage,rate=rate,index=i,method='POST' if i%5==0 else 'GET',
scheduled=due-origin,started='',finished=now-origin,latency_ms='',
schedule_lag_ms=round((now-due)*1000,3),status=0,instance='',ok=False,
error='generator_drop'))
else:
futures.append(pool.submit(request,i,due))
remainder=start+duration-time.monotonic()
if remainder>0:time.sleep(remainder)
for f in futures:f.result()
ended=time.monotonic()
file.flush()
if stage=='measure':
good=[r for r in rows if r['ok']]
latency=[r['latency_ms'] for r in good]
drops=sum(r['error']=='generator_drop' for r in rows)
summary={'target_rps':rate,'scheduled_seconds':duration,
'elapsed_with_drain_seconds':round(ended-start,3),'scheduled_requests':len(rows),
'sent':len(rows)-drops,'generator_drops':drops,'successes':len(good),
'errors':len(rows)-len(good)-drops,
'success_rps_with_drain':round(len(good)/(ended-start),3),
'sent_rps':round((len(rows)-drops)/duration,3),
'failure_pct_including_drops':round(100*(len(rows)-len(good))/len(rows),3),
'p50_ms_success':percentile(latency,.5),'p95_ms_success':percentile(latency,.95),
'p99_ms_success':percentile(latency,.99),
'p95_schedule_lag_ms':percentile([r['schedule_lag_ms'] for r in rows],.95),
'instances':sorted({r['instance'] for r in good})}
metadata['summary'].append(summary)
(output/'summary.json').write_text(json.dumps(metadata,indent=2),encoding='utf-8')
print(json.dumps(summary),flush=True)
phase.update(name='cooldown',rate=rate)
time.sleep(5)
finally:
stop_sample.set();monitor.join(timeout=25)
print('Results:',output,flush=True)
print('DONE: send summary.json and resources.jsonl for saturation analysis')
if __name__=='__main__':
try:main()
except Exception as exc:
print('FAIL:',str(exc));raise SystemExit(1)