69 lines
5.0 KiB
Python
69 lines
5.0 KiB
Python
"""Static research figures from recorded evidence only; no new measurements."""
|
||
import csv
|
||
import json
|
||
from pathlib import Path
|
||
import re
|
||
import statistics
|
||
import matplotlib
|
||
matplotlib.use('Agg')
|
||
import matplotlib.pyplot as plt
|
||
ROOT=Path(__file__).resolve().parents[1]
|
||
OUT=ROOT/'evidence/figures';OUT.mkdir(exist_ok=True)
|
||
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.spines.top':False,'axes.spines.right':False})
|
||
def save(fig,name):
|
||
fig.tight_layout()
|
||
fig.savefig(OUT/(name+'.png'),dpi=170)
|
||
fig.savefig(OUT/(name+'.svg'))
|
||
plt.close(fig)
|
||
def bands(ax,x,groups,label,color):
|
||
med=[statistics.median(v) for v in groups]
|
||
ax.errorbar(x,med,yerr=[[m-min(v) for m,v in zip(med,groups)],[max(v)-m for m,v in zip(med,groups)]],fmt='o-',capsize=4,label=label,color=color)
|
||
r=json.loads((ROOT/'evidence/reconcile.json').read_text(encoding='utf-8-sig'))
|
||
points=sorted({v['extra_objects'] for v in r['cost']})
|
||
groups=[[v for v in r['cost'] if v['extra_objects']==n] for n in points]
|
||
x=[v[0]['counts']['objects_scanned'] for v in groups]
|
||
fig,axes=plt.subplots(1,2,figsize=(10.5,4))
|
||
bands(axes[0],x,[[v['elapsed_seconds'] for v in vs] for vs in groups],'Медиана и min–max (3 повтора)','#286D8C')
|
||
axes[0].set(xlabel='Объектов в bucket',ylabel='Время dry-run, с',title='Стоимость полной сверки')
|
||
axes[0].legend(fontsize=8)
|
||
for field,label in [('head_requests','HEAD'),('get_requests','GET'),('list_requests','LIST')]:
|
||
axes[1].plot(x,[v[0]['counts'][field] for v in groups],'o-',label=label)
|
||
axes[1].set(xlabel='Объектов в bucket',ylabel='Число S3-запросов',title='Цена обхода: запросы')
|
||
axes[1].legend();save(fig,'reconcile-cost')
|
||
u=json.loads((ROOT/'evidence/upload-load.json').read_text(encoding='utf-8-sig'))
|
||
sizes=sorted({v['size_bytes'] for v in u['runs']}); x=[s/1048576 for s in sizes]
|
||
groups=[[v for v in u['runs'] if v['size_bytes']==s] for s in sizes]
|
||
fig,axes=plt.subplots(1,2,figsize=(10.5,4))
|
||
bands(axes[0],x,[[v['successful_rps'] for v in vs] for vs in groups],'3 партии × 10 клиентов','#286D8C')
|
||
axes[0].set(xlabel='PNG, MiB',ylabel='Успешных загрузок/с',title='Синхронный upload через API')
|
||
bands(axes[1],x,[[v['p95_seconds'] for v in vs] for vs in groups],'p95 успешных запросов','#B15E38')
|
||
axes[1].set(xlabel='PNG, MiB',ylabel='Задержка, с',title='p95 внутри партии (10 запросов)')
|
||
for ax in axes: ax.legend(fontsize=8)
|
||
save(fig,'upload-throughput')
|
||
def mib(value):
|
||
match=re.match(r'([0-9.]+)([A-Za-z]+)',value.strip())
|
||
amount=float(match[1]);unit=match[2]
|
||
return amount*{'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024,'kB':1000/1048576,'MB':1000000/1048576,'GB':1000000000/1048576}[unit]
|
||
fig,ax=plt.subplots(figsize=(10.5,4))
|
||
first=u['samples'][0]['time']
|
||
for name in sorted({c['Name'] for s in u['samples'] for c in s['containers'] if '-api-' in c['Name']}):
|
||
pairs=[(s['time']-first,mib(c['MemUsage'].split('/')[0])) for s in u['samples'] for c in s['containers'] if c['Name']==name]
|
||
ax.plot([a for a,b in pairs],[b for a,b in pairs],'o-',label=name)
|
||
ax.axhline(512,color='#B15E38',linestyle='--',label='Лимит одной API: 512 MiB')
|
||
ax.set(xlabel='Время от первой выборки, с',ylabel='Docker working set, MiB',title='Память API: прогрев, загрузки и проверка оригиналов')
|
||
ax.legend(fontsize=8);save(fig,'upload-memory')
|
||
with (ROOT/'evidence/upload-summary.csv').open('w',newline='',encoding='utf-8') as f:
|
||
writer=csv.DictWriter(f,fieldnames=list(u['runs'][0]));writer.writeheader();writer.writerows(u['runs'])
|
||
summary={'uploads':[], 'cost':[],'sampled_memory_max_mib':{},'kernel_memory_peak_mib':{}}
|
||
for size,vs in zip(sizes,groups):
|
||
summary['uploads'].append({'size_bytes':size,'successful_rps_median':statistics.median(v['successful_rps'] for v in vs),'p50_seconds_median':statistics.median(v['p50_seconds'] for v in vs),'p95_seconds_median':statistics.median(v['p95_seconds'] for v in vs),'p99_seconds_median':statistics.median(v['p99_seconds'] for v in vs),'errors':sum(v['errors'] for v in vs),'attempts':sum(v['attempts'] for v in vs),'elapsed_seconds':sum(v['elapsed_seconds'] for v in vs)})
|
||
for n in points:
|
||
vs=[v for v in r['cost'] if v['extra_objects']==n]
|
||
summary['cost'].append({'objects':vs[0]['counts']['objects_scanned'],'elapsed_median':statistics.median(v['elapsed_seconds'] for v in vs),'elapsed_min':min(v['elapsed_seconds'] for v in vs),'elapsed_max':max(v['elapsed_seconds'] for v in vs),'counts':vs[0]['counts']})
|
||
for s in u['samples']:
|
||
for c in s['containers']:
|
||
name=c['Name'];summary['sampled_memory_max_mib'][name]=max(summary['sampled_memory_max_mib'].get(name,0),mib(c['MemUsage'].split('/')[0]))
|
||
for c in u.get('cgroup_after',[]):summary['kernel_memory_peak_mib'][c['id']]=int(c['memory.peak'])/1048576
|
||
(ROOT/'evidence/summary.json').write_text(json.dumps(summary,indent=2),encoding='utf-8')
|
||
print(json.dumps(summary,indent=2))
|