Files
mtusi2026lab3-fork/scripts/research.py
T

244 lines
15 KiB
Python

"""Grade-5 track 1: controlled scaling, saturation sweep and pool tuning."""
import argparse
import csv
import datetime as dt
import io
import json
import os
from pathlib import Path
import platform
import statistics
import subprocess
import threading
import time
from manage import ROOT, docker, ready
def sql(query):
return json.loads(docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-At','-c',query,capture=True))
def db_stats():
return sql("SELECT row_to_json(s) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_inserted,deadlocks FROM pg_stat_database WHERE datname=current_database()) s")
def lb_stats():
text = docker('compose','exec','-T','lb','wget','-q','-O','-','http://127.0.0.1:8404/stats;csv',capture=True)
rows = list(csv.DictReader(io.StringIO(text.removeprefix('# '))))
return [{k:r.get(k) for k in ('svname','status','qcur','qmax','scur','smax','econ','eresp','stot')} for r in rows if r.get('pxname')=='api']
def configure(n, cpu, memory, pool):
env = dict(os.environ, DB_CPUS='1.0', API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=str(memory)+'m',
DB_POOL_MAX=str(pool), POOL_METRICS='1', REQUEST_LOG='0')
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api='+str(n),'api',env=env)
ready(n)
return env
def measure(config, duration, warmup):
began = dt.datetime.now(dt.timezone.utc).isoformat()
before, lb_before = db_stats(), lb_stats()
samples = []
finished = threading.Event()
sample_errors = []
def monitor():
while not finished.is_set():
try:
ids = docker('ps','-q','--filter','label=com.docker.compose.project=mtusi-lab3',capture=True).split()
ids += docker('ps','-q','--filter','label=mtusi.lab3.role=load',capture=True).split()
text = docker('stats','--no-stream','--format','{{json .}}',*ids,capture=True)
stats = [json.loads(line) for line in text.splitlines() if line.strip()]
waits = sql("SELECT coalesce(json_object_agg(kind,n),'{}') FROM (SELECT coalesce(wait_event_type,'CPU/runnable') kind,count(*) n FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active' GROUP BY 1) s")
samples.append({'time':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':stats,'db_active_waits':waits})
except (subprocess.CalledProcessError, ValueError) as error:
sample_errors.append(type(error).__name__)
finished.wait(.5)
thread = threading.Thread(target=monitor)
thread.start()
try:
output = docker('run','--rm','--network','mtusi-lab3_default','--cpus','2','--memory','256m',
'--label','mtusi.lab3.role=load','mtusi-messenger:lab3-load',
'-duration',str(duration)+'s','-warmup',str(warmup)+'s',
'-concurrency',str(config['concurrency']),capture=True)
result = json.loads(output)
finally:
finished.set()
thread.join()
after = db_stats()
logs = docker('compose','logs','--no-log-prefix','--since',began,'api',capture=True)
pools = {}
for line in logs.splitlines():
try:
item = json.loads(line)
except ValueError:
continue
if item.get('msg')=='pool':
pools.setdefault(item['instance'],[]).append(item)
pool_summary = {}
for instance, values in pools.items():
pool_summary[instance] = {'max_acquired':max(v['acquired'] for v in values),'max_connections':values[-1]['max'],
'empty_acquires_delta':values[-1]['empty_acquires']-values[0]['empty_acquires'],
'acquire_seconds_delta':(values[-1]['acquire_ns']-values[0]['acquire_ns'])/1e9}
if len(result['instances']) != config['replicas']:
raise RuntimeError('load did not reach the configured number of replicas')
if not samples or not pool_summary:
raise RuntimeError('missing CPU/DB/pool observations')
return dict(config, result=result, samples=samples, sample_errors=sample_errors, pool_stats=pool_summary,
db_delta={k:after[k]-before[k] for k in before},lb_before=lb_before,lb_after=lb_stats())
def summarize(rows, folder):
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
grouped = {}
for row in rows:
key = (row['series'],row['replicas'],row['concurrency'],row['pool'])
grouped.setdefault(key,[]).append(row)
summaries = []
for key, values in sorted(grouped.items()):
rps = [r['result']['successful_rps'] for r in values]
summaries.append({'series':key[0],'replicas':key[1],'concurrency':key[2],'pool':key[3],
'rps_median':statistics.median(rps),'rps_min':min(rps),'rps_max':max(rps),
'p95_ms_median':statistics.median(r['result']['latency_ms']['p95'] for r in values),
'failed_requests':sum(r['result']['failed_requests'] for r in values),
'successful_requests':sum(r['result']['successful_requests'] for r in values)})
with (folder/'research.csv').open('w',newline='',encoding='utf-8') as output:
writer = csv.DictWriter(output,fieldnames=list(summaries[0]))
writer.writeheader()
writer.writerows(summaries)
fig, axes = plt.subplots(1,2,figsize=(11,4),layout='constrained')
labels = {'fixed':'fixed total: 2 CPU / 1024 MiB','growing':'growing: 1 CPU / 512 MiB per API'}
for series in ('fixed','growing'):
points = sorted((s for s in summaries if s['series']==series),key=lambda p:p['replicas'])
axes[0].errorbar([p['replicas'] for p in points],[p['rps_median'] for p in points],
yerr=[[p['rps_median']-p['rps_min'] for p in points],[p['rps_max']-p['rps_median'] for p in points]],marker='o',capsize=4,label=labels[series])
axes[1].plot([p['replicas'] for p in points],[p['p95_ms_median'] for p in points],marker='o',label=labels[series])
for axis in axes:
axis.set_xlabel('API replicas')
axis.set_xticks([1,2,4])
axis.grid(alpha=.25)
axis.legend(fontsize=8)
axis.set_ylim(bottom=0)
fig.suptitle('3 repetitions; 15 s after 3 s warm-up; 32 closed-loop clients')
axes[0].set_ylabel('Successful HTTP confirmations / second')
axes[0].set_title('Median and range across repetitions')
axes[1].set_ylabel('p95 of successful requests, ms')
fig.savefig(folder/'scaling.png',dpi=160)
plt.close(fig)
sweep = sorted((s for s in summaries if s['series']=='saturation'),key=lambda p:p['concurrency'])
fig, axes = plt.subplots(1,3,figsize=(12,4),layout='constrained')
clients = [p['concurrency'] for p in sweep]
axes[0].errorbar(clients,[p['rps_median'] for p in sweep],
yerr=[[p['rps_median']-p['rps_min'] for p in sweep],[p['rps_max']-p['rps_median'] for p in sweep]],marker='o',capsize=4)
axes[0].set_ylabel('Successful confirmations / second')
axes[1].plot(clients,[p['p95_ms_median'] for p in sweep],marker='o')
axes[1].set_ylabel('p95 of successful requests, ms')
axes[2].plot(clients,[100*p['failed_requests']/(p['failed_requests']+p['successful_requests']) for p in sweep],marker='o',color='tab:red')
axes[2].set_ylabel('Pooled failed attempts, %')
for axis in axes:
axis.set_xscale('log',base=2)
axis.set_xticks(clients,labels=[str(c) for c in clients])
axis.set_xlabel('Concurrent clients')
axis.set_ylim(bottom=0)
axis.grid(alpha=.25)
fig.suptitle('Saturation: 4 API replicas; 3 repetitions; 15 s measurements')
fig.savefig(folder/'saturation.png',dpi=160)
plt.close(fig)
tuning = {s['pool']:s for s in summaries if s['series']=='pool-tuning' and s['concurrency']==32}
if set(tuning)!={1,8}:
raise RuntimeError('controlled pool tuning at concurrency 32 is missing; continue with --resume')
improvement = tuning[8]['rps_median']/tuning[1]['rps_median']-1
confirmed = improvement>.05 and tuning[8]['failed_requests']<=tuning[1]['failed_requests']
lines = ['# Измерения трека 1','',
'Fixed: суммарно 2 CPU / 1024 MiB API, суммарный пул 8. Growing: 1 CPU / 512 MiB и пул 8 на каждую API. PostgreSQL 1 CPU / 512 MiB, LB 0.5 CPU / 128 MiB, Valkey 0.5 CPU / 256 MiB, генератор 2 CPU / 256 MiB.','',
'| Серия | API | Клиенты | Пул/реплика | RPS median [min;max] | p95 ms | Ошибки |',
'|---|---:|---:|---:|---:|---:|---:|']
for s in summaries:
lines.append('| {series} | {replicas} | {concurrency} | {pool} | {rps_median:.1f} [{rps_min:.1f}; {rps_max:.1f}] | {p95_ms_median:.1f} | {failed_requests} |'.format(**s))
lines += ['',f'Контролируемая настройка пула 1 → 8 (2 API, 32 клиента): изменение median RPS {improvement:+.1%}. Критерий подтверждения: >5% без увеличения общего числа ошибок. Подтверждено: {confirmed}.',
'', 'Диагностика: research.json содержит CPU/RAM каждого контейнера, активные ожидания PostgreSQL, счётчики ожидания пула и очереди/ошибки HAProxy. Pool-tuning — отдельный намеренно ограниченный сценарий; он не выдаётся за дефект исходного пула 8.',
'', 'Нужно интерпретировать насыщение по совместным данным: занятый пул и рост empty_acquires указывают на очередь пула; PostgreSQL около 100% при лимите 1 CPU и отсутствие прироста RPS указывают на CPU БД; очередь/CPU LB — на LB. Один плоский график не доказывает причину.',
'', 'Ограничения: один Docker Desktop/хост; closed-loop и coordinated omission; p95 только успехов; контейнерные CPU/RAM и PostgreSQL-счётчики включают прогрев; история растёт; дисковый кэш между сериями не сбрасывается; max RPS промышленной системы не заявляется.','']
(folder/'research.md').write_text('\n'.join(lines),encoding='utf-8')
return confirmed
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--duration',type=int,default=15)
p.add_argument('--warmup',type=int,default=3)
p.add_argument('--repeats',type=int,default=3)
p.add_argument('--resume',action='store_true',help='reuse completed runs with identical measurement parameters')
p.add_argument('--include-overload',action='store_true',help='also reproduce the rejected 64-client pool experiment')
args = p.parse_args()
if args.duration<10 or args.warmup<2 or args.repeats<3:
p.error('research needs duration>=10, warmup>=2, repeats>=3')
import matplotlib # fail before altering running services if dependencies are absent
folder = ROOT/'evidence'
folder.mkdir(exist_ok=True)
metadata = {'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'host':platform.platform(),
'host_cpu':platform.processor(),'logical_cpus':os.cpu_count(),
'docker':json.loads(docker('info','--format','{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"version":"{{.ServerVersion}}","os":"{{.OperatingSystem}}"}',capture=True)),
'profile':vars(args),'hypothesis':'Shared DB/LB or pool eventually limits scaling; more API replicas alone cannot remove that limit.'}
rows = []
saved = folder/'research.json'
if args.resume and saved.exists():
previous = json.loads(saved.read_text(encoding='utf-8'))
old_profile = previous['metadata']['profile']
if any(old_profile[k]!=getattr(args,k) for k in ('duration','warmup','repeats')):
p.error('saved measurement parameters differ; cannot resume')
metadata = previous['metadata']
rows = previous['runs']
# Upgrade early evidence where the pool telemetry shadowed its setting.
for row in rows:
if isinstance(row.get('pool'),dict):
row['pool_stats'] = row['pool']
row['pool'] = next(iter(row['pool_stats'].values()))['max_connections']
if row['series']=='pool-tuning' and row['concurrency']==64:
row['series']='pool-overload'
print('Reusing %d completed measurements' % len(rows),flush=True)
try:
for repeat in range(args.repeats):
configs = []
for series in ('fixed','growing'):
order = [1,2,4]
order = order[repeat%3:]+order[:repeat%3]
for n in order:
configs.append({'series':series,'replicas':n,'cpu_per_api':2/n if series=='fixed' else 1,
'memory_mib_per_api':1024//n if series=='fixed' else 512,
'pool':8//n if series=='fixed' else 8,'concurrency':32,'repeat':repeat+1})
for pool in ([1,8] if repeat%2==0 else [8,1]):
configs.append({'series':'pool-tuning','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':32,'repeat':repeat+1})
if args.include_overload:
for pool in ([1,8] if repeat%2==0 else [8,1]):
configs.append({'series':'pool-overload','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':64,'repeat':repeat+1})
for c in ([1,8,32,64,128] if repeat%2==0 else [128,64,32,8,1]):
configs.append({'series':'saturation','replicas':4,'cpu_per_api':1,'memory_mib_per_api':512,'pool':8,'concurrency':c,'repeat':repeat+1})
previous = None
for config in configs:
if any(all(row[k]==config[k] for k in ('series','replicas','concurrency','pool','repeat')) for row in rows):
continue
state = (config['replicas'],config['cpu_per_api'],config['memory_mib_per_api'],config['pool'])
if state!=previous:
configure(*state)
previous = state
print('Measuring '+json.dumps(config),flush=True)
rows.append(measure(config,args.duration,args.warmup))
(folder/'research.json').write_text(json.dumps({'metadata':metadata,'runs':rows},indent=2)+'\n',encoding='utf-8')
confirmed = summarize(rows,folder)
if not confirmed:
raise RuntimeError('pool tuning hypothesis was not confirmed; see research.md before claiming grade 5')
finally:
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api=2','api')
ready(2)
print('PASS: repeated scaling/saturation series and pool improvement measured; see evidence/research.md')
if __name__=='__main__':
main()