Files
mtusi2026lab3/scripts/summarize-research.py
T
2026-11-07 10:41:21 +03:00

87 lines
6.0 KiB
Python

#!/usr/bin/env python3
"""Reduce raw runs and cgroup samples; generate standard scientific plots."""
import csv,json,statistics,math
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1];FOLDER=ROOT/'evidence/research'
def metrics(path,result):
if not path.exists():return {}
rows=[json.loads(s) for s in path.read_text().splitlines() if s.strip()]
start=result.get('measurement_started_unix',rows[0]['unix_time'] if rows else 0)
end=start+result['elapsed_including_drain_seconds']
# Include only intervals with both endpoints inside the measured phase.
previous={};cpu={};throttled={};mem={};wait=[];backlog=[];dbwaits=set();generator_cpu=[];generator_rss=[];total_memory_peak={}
for row in rows:
t=row['unix_time'];inside=start<=t<=end;row_memory={}
if inside and row.get('generator_host'):
generator_cpu.append(row['generator_host']['cpu_percent']/100);generator_rss.append(row['generator_host']['rss_kib'])
for c in row['containers']:
name=c['name'];service=c['service'];g=c.get('cgroup',{});prev=previous.get(name)
if 'usage_usec' not in g:continue
if inside and prev and prev[0]>=start and t>prev[0]:
dt=t-prev[0];cpu.setdefault(service,[]).append((g.get('usage_usec',0)-prev[1].get('usage_usec',0))/1e6/dt)
periods=g.get('nr_periods',0)-prev[1].get('nr_periods',0)
if periods>0:throttled.setdefault(service,[]).append((g.get('nr_throttled',0)-prev[1].get('nr_throttled',0))/periods)
pool=c.get('pool');oldpool=prev[2]
if pool and oldpool:
count=pool['acquire_count']-oldpool['acquire_count']
if count>0:wait.append((pool['acquire_duration_ns']-oldpool['acquire_duration_ns'])/count/1e6)
if inside:
mem[service]=max(mem.get(service,0),g.get('memory_bytes',0))
row_memory[service]=row_memory.get(service,0)+g.get('memory_bytes',0)
for b in c.get('backends',[]):
if b['svname']=='BACKEND':backlog.append(int(b['qcur'] or 0))
for w in c.get('database',{}).get('waits',[]):
if w['state']=='active' and w['wait_event_type']:dbwaits.add(w['wait_event_type'])
previous[name]=(t,g,c.get('pool'))
for service,value in row_memory.items():total_memory_peak[service]=max(total_memory_peak.get(service,0),value)
return {'cpu_cores_mean_per_container':{k:statistics.mean(v) for k,v in cpu.items()},'throttled_period_ratio_mean':{k:statistics.mean(v) for k,v in throttled.items()},'memory_peak_bytes_per_container':mem,'service_total_memory_peak_bytes':total_memory_peak,'pool_acquire_ms_mean':statistics.mean(wait) if wait else None,'lb_queue_peak':max(backlog,default=0),'database_active_wait_types':sorted(dbwaits),'samples':len(rows),'generator_cpu_cores_mean':statistics.mean(generator_cpu) if generator_cpu else None,'generator_rss_peak_kib':max(generator_rss,default=0)}
def main():
allrows=[]
for p in sorted(FOLDER.glob('*.json')):
r=json.loads(p.read_text())
if 'target_rps' not in r:continue
label=r['label']
if not any(label.startswith(x) for x in ('fixed-','growing-','before-','after-','lbprobe-')):continue
r['metrics']=metrics(p.with_suffix('.metrics.jsonl'),r)
allrows.append(r)
(FOLDER/'runs.json').write_text(json.dumps(allrows,indent=2)+'\n')
groups={}
for r in allrows:
group=r['label'].rsplit('-n',1)[0];groups.setdefault(group,[]).append(r)
summary=[]
for group,rows in sorted(groups.items()):
summary.append({'case':group,'runs':len(rows),'target_rps':rows[0]['target_rps'],'median_successful_rps':statistics.median(r['successful_rps'] for r in rows),'min_successful_rps':min(r['successful_rps'] for r in rows),'max_successful_rps':max(r['successful_rps'] for r in rows),'median_error_ratio':statistics.median(r['error_ratio_including_drops'] for r in rows),**{f'median_{q}_ms':statistics.median(r['successful_latency_ms'].get(q,float('nan')) for r in rows) for q in ('p50','p95','p99')}})
(FOLDER/'summary.json').write_text(json.dumps(summary,indent=2)+'\n')
with (FOLDER/'summary.csv').open('w') as f:
if summary:w=csv.DictWriter(f,fieldnames=summary[0],lineterminator="\n");w.writeheader();w.writerows(summary)
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
plt.rcParams.update({'font.size':10,'axes.grid':True,'grid.alpha':.25})
fig,axes=plt.subplots(2,2,figsize=(11,7),constrained_layout=True)
for col,series in enumerate(('fixed','growing')):
for n in (1,2,4):
rs=sorted((r for r in summary if r['case'].startswith(f'{series}-{n}-')),key=lambda r:r['target_rps'])
if not rs:continue
x=[r['target_rps'] for r in rs]
line,=axes[0,col].plot(x,[r['median_successful_rps'] for r in rs],'o-',label=f'{n} API')
axes[0,col].fill_between(x,[r['min_successful_rps'] for r in rs],[r['max_successful_rps'] for r in rs],color=line.get_color(),alpha=.12)
axes[1,col].plot(x,[100*r['median_error_ratio'] for r in rs],'o-',label=f'{n} API')
axes[0,col].set_title('Fixed total: 2 CPU / 1 GiB' if series=='fixed' else 'Growing: 1 CPU / 512 MiB per API')
for ax in axes[:,col]:ax.set_xscale('log');ax.set_xlabel('Scheduled requests / second');ax.legend()
axes[0,col].set_ylabel('Successful requests / second');axes[1,col].set_ylabel('Errors + generator drops (%)')
fig.suptitle('Medians of 3 runs; shaded throughput bands show min-max')
fig.savefig(FOLDER/'scaling.png',dpi=180);plt.close(fig)
fig,axes=plt.subplots(1,2,figsize=(10,4),constrained_layout=True)
for version in ('before','after'):
rs=sorted((r for r in summary if r['case'].startswith(version+'-2-')),key=lambda r:r['target_rps'])
if rs:
axes[0].plot([r['target_rps'] for r in rs],[r['median_successful_rps'] for r in rs],'o-',label=version)
axes[1].plot([r['target_rps'] for r in rs],[100*r['median_error_ratio'] for r in rs],'o-',label=version)
for ax in axes:ax.set_xlabel('Scheduled requests / second');ax.legend()
axes[0].set_ylabel('Successful requests / second');axes[1].set_ylabel('Errors + drops (%)')
fig.suptitle('Token fingerprint optimization: same 2 API, CPU, RAM and workload')
fig.savefig(FOLDER/'optimization.png',dpi=180);plt.close(fig)
print(f'Summarized {len(allrows)} runs in {len(summary)} cases')
if __name__=='__main__':main()