Завершить трек 2: доказать согласованность хранилищ и оформить графики и отчёт

This commit is contained in:
2026-10-30 17:54:00 +03:00
parent 9dded977a5
commit 6c87ed2eba
20 changed files with 6831 additions and 21 deletions
+5 -1
View File
@@ -12,9 +12,13 @@ url_file = folder / 'database_url'
if not url_file.exists():
password = password_file.read_text(encoding='utf-8').strip()
url_file.write_text(f'postgresql://messenger:{password}@db:5432/messenger', encoding='utf-8')
print('Локальные секреты готовы; значения не выводятся.')
for name in ('s3_root_user', 's3_root_password', 's3_access_key', 's3_secret_key'):
target = folder / name
if not target.exists():
target.write_text(secrets.token_hex(24), encoding='utf-8')
env_file = root / '.env'
if not env_file.exists():
env_file.write_bytes((root / 'lab4.env.example').read_bytes())
print('Локальные настройки и секреты готовы; существующие файлы сохранены.')
+68
View File
@@ -0,0 +1,68 @@
"""Static research figures from recorded evidence only; no new measurements."""
import csv
import json
from pathlib import Path
import re
import statistics
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT=Path(__file__).resolve().parents[1]
OUT=ROOT/'evidence/figures';OUT.mkdir(exist_ok=True)
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.spines.top':False,'axes.spines.right':False})
def save(fig,name):
fig.tight_layout()
fig.savefig(OUT/(name+'.png'),dpi=170)
fig.savefig(OUT/(name+'.svg'))
plt.close(fig)
def bands(ax,x,groups,label,color):
med=[statistics.median(v) for v in groups]
ax.errorbar(x,med,yerr=[[m-min(v) for m,v in zip(med,groups)],[max(v)-m for m,v in zip(med,groups)]],fmt='o-',capsize=4,label=label,color=color)
r=json.loads((ROOT/'evidence/reconcile.json').read_text(encoding='utf-8-sig'))
points=sorted({v['extra_objects'] for v in r['cost']})
groups=[[v for v in r['cost'] if v['extra_objects']==n] for n in points]
x=[v[0]['counts']['objects_scanned'] for v in groups]
fig,axes=plt.subplots(1,2,figsize=(10.5,4))
bands(axes[0],x,[[v['elapsed_seconds'] for v in vs] for vs in groups],'Медиана и min–max (3 повтора)','#286D8C')
axes[0].set(xlabel='Объектов в bucket',ylabel='Время dry-run, с',title='Стоимость полной сверки')
axes[0].legend(fontsize=8)
for field,label in [('head_requests','HEAD'),('get_requests','GET'),('list_requests','LIST')]:
axes[1].plot(x,[v[0]['counts'][field] for v in groups],'o-',label=label)
axes[1].set(xlabel='Объектов в bucket',ylabel='Число S3-запросов',title='Цена обхода: запросы')
axes[1].legend();save(fig,'reconcile-cost')
u=json.loads((ROOT/'evidence/upload-load.json').read_text(encoding='utf-8-sig'))
sizes=sorted({v['size_bytes'] for v in u['runs']}); x=[s/1048576 for s in sizes]
groups=[[v for v in u['runs'] if v['size_bytes']==s] for s in sizes]
fig,axes=plt.subplots(1,2,figsize=(10.5,4))
bands(axes[0],x,[[v['successful_rps'] for v in vs] for vs in groups],'3 партии × 10 клиентов','#286D8C')
axes[0].set(xlabel='PNG, MiB',ylabel='Успешных загрузок/с',title='Синхронный upload через API')
bands(axes[1],x,[[v['p95_seconds'] for v in vs] for vs in groups],'p95 успешных запросов','#B15E38')
axes[1].set(xlabel='PNG, MiB',ylabel='Задержка, с',title='p95 внутри партии (10 запросов)')
for ax in axes: ax.legend(fontsize=8)
save(fig,'upload-throughput')
def mib(value):
match=re.match(r'([0-9.]+)([A-Za-z]+)',value.strip())
amount=float(match[1]);unit=match[2]
return amount*{'B':1/1048576,'KiB':1/1024,'MiB':1,'GiB':1024,'kB':1000/1048576,'MB':1000000/1048576,'GB':1000000000/1048576}[unit]
fig,ax=plt.subplots(figsize=(10.5,4))
first=u['samples'][0]['time']
for name in sorted({c['Name'] for s in u['samples'] for c in s['containers'] if '-api-' in c['Name']}):
pairs=[(s['time']-first,mib(c['MemUsage'].split('/')[0])) for s in u['samples'] for c in s['containers'] if c['Name']==name]
ax.plot([a for a,b in pairs],[b for a,b in pairs],'o-',label=name)
ax.axhline(512,color='#B15E38',linestyle='--',label='Лимит одной API: 512 MiB')
ax.set(xlabel='Время от первой выборки, с',ylabel='Docker working set, MiB',title='Память API: прогрев, загрузки и проверка оригиналов')
ax.legend(fontsize=8);save(fig,'upload-memory')
with (ROOT/'evidence/upload-summary.csv').open('w',newline='',encoding='utf-8') as f:
writer=csv.DictWriter(f,fieldnames=list(u['runs'][0]));writer.writeheader();writer.writerows(u['runs'])
summary={'uploads':[], 'cost':[],'sampled_memory_max_mib':{},'kernel_memory_peak_mib':{}}
for size,vs in zip(sizes,groups):
summary['uploads'].append({'size_bytes':size,'successful_rps_median':statistics.median(v['successful_rps'] for v in vs),'p50_seconds_median':statistics.median(v['p50_seconds'] for v in vs),'p95_seconds_median':statistics.median(v['p95_seconds'] for v in vs),'p99_seconds_median':statistics.median(v['p99_seconds'] for v in vs),'errors':sum(v['errors'] for v in vs),'attempts':sum(v['attempts'] for v in vs),'elapsed_seconds':sum(v['elapsed_seconds'] for v in vs)})
for n in points:
vs=[v for v in r['cost'] if v['extra_objects']==n]
summary['cost'].append({'objects':vs[0]['counts']['objects_scanned'],'elapsed_median':statistics.median(v['elapsed_seconds'] for v in vs),'elapsed_min':min(v['elapsed_seconds'] for v in vs),'elapsed_max':max(v['elapsed_seconds'] for v in vs),'counts':vs[0]['counts']})
for s in u['samples']:
for c in s['containers']:
name=c['Name'];summary['sampled_memory_max_mib'][name]=max(summary['sampled_memory_max_mib'].get(name,0),mib(c['MemUsage'].split('/')[0]))
for c in u.get('cgroup_after',[]):summary['kernel_memory_peak_mib'][c['id']]=int(c['memory.peak'])/1048576
(ROOT/'evidence/summary.json').write_text(json.dumps(summary,indent=2),encoding='utf-8')
print(json.dumps(summary,indent=2))