244 lines
15 KiB
Python
244 lines
15 KiB
Python
"""Grade-5 track 1: controlled scaling, saturation sweep and pool tuning."""
|
|
import argparse
|
|
import csv
|
|
import datetime as dt
|
|
import io
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import statistics
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
|
|
from manage import ROOT, docker, ready
|
|
|
|
|
|
def sql(query):
|
|
return json.loads(docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-At','-c',query,capture=True))
|
|
|
|
|
|
def db_stats():
|
|
return sql("SELECT row_to_json(s) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_inserted,deadlocks FROM pg_stat_database WHERE datname=current_database()) s")
|
|
|
|
|
|
def lb_stats():
|
|
text = docker('compose','exec','-T','lb','wget','-q','-O','-','http://127.0.0.1:8404/stats;csv',capture=True)
|
|
rows = list(csv.DictReader(io.StringIO(text.removeprefix('# '))))
|
|
return [{k:r.get(k) for k in ('svname','status','qcur','qmax','scur','smax','econ','eresp','stot')} for r in rows if r.get('pxname')=='api']
|
|
|
|
|
|
def configure(n, cpu, memory, pool):
|
|
env = dict(os.environ, DB_CPUS='1.0', API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=str(memory)+'m',
|
|
DB_POOL_MAX=str(pool), POOL_METRICS='1', REQUEST_LOG='0')
|
|
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api='+str(n),'api',env=env)
|
|
ready(n)
|
|
return env
|
|
|
|
|
|
def measure(config, duration, warmup):
|
|
began = dt.datetime.now(dt.timezone.utc).isoformat()
|
|
before, lb_before = db_stats(), lb_stats()
|
|
samples = []
|
|
finished = threading.Event()
|
|
sample_errors = []
|
|
|
|
def monitor():
|
|
while not finished.is_set():
|
|
try:
|
|
ids = docker('ps','-q','--filter','label=com.docker.compose.project=mtusi-lab3',capture=True).split()
|
|
ids += docker('ps','-q','--filter','label=mtusi.lab3.role=load',capture=True).split()
|
|
text = docker('stats','--no-stream','--format','{{json .}}',*ids,capture=True)
|
|
stats = [json.loads(line) for line in text.splitlines() if line.strip()]
|
|
waits = sql("SELECT coalesce(json_object_agg(kind,n),'{}') FROM (SELECT coalesce(wait_event_type,'CPU/runnable') kind,count(*) n FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active' GROUP BY 1) s")
|
|
samples.append({'time':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':stats,'db_active_waits':waits})
|
|
except (subprocess.CalledProcessError, ValueError) as error:
|
|
sample_errors.append(type(error).__name__)
|
|
finished.wait(.5)
|
|
|
|
thread = threading.Thread(target=monitor)
|
|
thread.start()
|
|
try:
|
|
output = docker('run','--rm','--network','mtusi-lab3_default','--cpus','2','--memory','256m',
|
|
'--label','mtusi.lab3.role=load','mtusi-messenger:lab3-load',
|
|
'-duration',str(duration)+'s','-warmup',str(warmup)+'s',
|
|
'-concurrency',str(config['concurrency']),capture=True)
|
|
result = json.loads(output)
|
|
finally:
|
|
finished.set()
|
|
thread.join()
|
|
after = db_stats()
|
|
logs = docker('compose','logs','--no-log-prefix','--since',began,'api',capture=True)
|
|
pools = {}
|
|
for line in logs.splitlines():
|
|
try:
|
|
item = json.loads(line)
|
|
except ValueError:
|
|
continue
|
|
if item.get('msg')=='pool':
|
|
pools.setdefault(item['instance'],[]).append(item)
|
|
pool_summary = {}
|
|
for instance, values in pools.items():
|
|
pool_summary[instance] = {'max_acquired':max(v['acquired'] for v in values),'max_connections':values[-1]['max'],
|
|
'empty_acquires_delta':values[-1]['empty_acquires']-values[0]['empty_acquires'],
|
|
'acquire_seconds_delta':(values[-1]['acquire_ns']-values[0]['acquire_ns'])/1e9}
|
|
if len(result['instances']) != config['replicas']:
|
|
raise RuntimeError('load did not reach the configured number of replicas')
|
|
if not samples or not pool_summary:
|
|
raise RuntimeError('missing CPU/DB/pool observations')
|
|
return dict(config, result=result, samples=samples, sample_errors=sample_errors, pool_stats=pool_summary,
|
|
db_delta={k:after[k]-before[k] for k in before},lb_before=lb_before,lb_after=lb_stats())
|
|
|
|
|
|
def summarize(rows, folder):
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
grouped = {}
|
|
for row in rows:
|
|
key = (row['series'],row['replicas'],row['concurrency'],row['pool'])
|
|
grouped.setdefault(key,[]).append(row)
|
|
summaries = []
|
|
for key, values in sorted(grouped.items()):
|
|
rps = [r['result']['successful_rps'] for r in values]
|
|
summaries.append({'series':key[0],'replicas':key[1],'concurrency':key[2],'pool':key[3],
|
|
'rps_median':statistics.median(rps),'rps_min':min(rps),'rps_max':max(rps),
|
|
'p95_ms_median':statistics.median(r['result']['latency_ms']['p95'] for r in values),
|
|
'failed_requests':sum(r['result']['failed_requests'] for r in values),
|
|
'successful_requests':sum(r['result']['successful_requests'] for r in values)})
|
|
with (folder/'research.csv').open('w',newline='',encoding='utf-8') as output:
|
|
writer = csv.DictWriter(output,fieldnames=list(summaries[0]))
|
|
writer.writeheader()
|
|
writer.writerows(summaries)
|
|
fig, axes = plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
|
labels = {'fixed':'fixed total: 2 CPU / 1024 MiB','growing':'growing: 1 CPU / 512 MiB per API'}
|
|
for series in ('fixed','growing'):
|
|
points = sorted((s for s in summaries if s['series']==series),key=lambda p:p['replicas'])
|
|
axes[0].errorbar([p['replicas'] for p in points],[p['rps_median'] for p in points],
|
|
yerr=[[p['rps_median']-p['rps_min'] for p in points],[p['rps_max']-p['rps_median'] for p in points]],marker='o',capsize=4,label=labels[series])
|
|
axes[1].plot([p['replicas'] for p in points],[p['p95_ms_median'] for p in points],marker='o',label=labels[series])
|
|
for axis in axes:
|
|
axis.set_xlabel('API replicas')
|
|
axis.set_xticks([1,2,4])
|
|
axis.grid(alpha=.25)
|
|
axis.legend(fontsize=8)
|
|
axis.set_ylim(bottom=0)
|
|
fig.suptitle('3 repetitions; 15 s after 3 s warm-up; 32 closed-loop clients')
|
|
axes[0].set_ylabel('Successful HTTP confirmations / second')
|
|
axes[0].set_title('Median and range across repetitions')
|
|
axes[1].set_ylabel('p95 of successful requests, ms')
|
|
fig.savefig(folder/'scaling.png',dpi=160)
|
|
plt.close(fig)
|
|
sweep = sorted((s for s in summaries if s['series']=='saturation'),key=lambda p:p['concurrency'])
|
|
fig, axes = plt.subplots(1,3,figsize=(12,4),layout='constrained')
|
|
clients = [p['concurrency'] for p in sweep]
|
|
axes[0].errorbar(clients,[p['rps_median'] for p in sweep],
|
|
yerr=[[p['rps_median']-p['rps_min'] for p in sweep],[p['rps_max']-p['rps_median'] for p in sweep]],marker='o',capsize=4)
|
|
axes[0].set_ylabel('Successful confirmations / second')
|
|
axes[1].plot(clients,[p['p95_ms_median'] for p in sweep],marker='o')
|
|
axes[1].set_ylabel('p95 of successful requests, ms')
|
|
axes[2].plot(clients,[100*p['failed_requests']/(p['failed_requests']+p['successful_requests']) for p in sweep],marker='o',color='tab:red')
|
|
axes[2].set_ylabel('Pooled failed attempts, %')
|
|
for axis in axes:
|
|
axis.set_xscale('log',base=2)
|
|
axis.set_xticks(clients,labels=[str(c) for c in clients])
|
|
axis.set_xlabel('Concurrent clients')
|
|
axis.set_ylim(bottom=0)
|
|
axis.grid(alpha=.25)
|
|
fig.suptitle('Saturation: 4 API replicas; 3 repetitions; 15 s measurements')
|
|
fig.savefig(folder/'saturation.png',dpi=160)
|
|
plt.close(fig)
|
|
tuning = {s['pool']:s for s in summaries if s['series']=='pool-tuning' and s['concurrency']==32}
|
|
if set(tuning)!={1,8}:
|
|
raise RuntimeError('controlled pool tuning at concurrency 32 is missing; continue with --resume')
|
|
improvement = tuning[8]['rps_median']/tuning[1]['rps_median']-1
|
|
confirmed = improvement>.05 and tuning[8]['failed_requests']<=tuning[1]['failed_requests']
|
|
lines = ['# Измерения трека 1','',
|
|
'Fixed: суммарно 2 CPU / 1024 MiB API, суммарный пул 8. Growing: 1 CPU / 512 MiB и пул 8 на каждую API. PostgreSQL 1 CPU / 512 MiB, LB 0.5 CPU / 128 MiB, Valkey 0.5 CPU / 256 MiB, генератор 2 CPU / 256 MiB.','',
|
|
'| Серия | API | Клиенты | Пул/реплика | RPS median [min;max] | p95 ms | Ошибки |',
|
|
'|---|---:|---:|---:|---:|---:|---:|']
|
|
for s in summaries:
|
|
lines.append('| {series} | {replicas} | {concurrency} | {pool} | {rps_median:.1f} [{rps_min:.1f}; {rps_max:.1f}] | {p95_ms_median:.1f} | {failed_requests} |'.format(**s))
|
|
lines += ['',f'Контролируемая настройка пула 1 → 8 (2 API, 32 клиента): изменение median RPS {improvement:+.1%}. Критерий подтверждения: >5% без увеличения общего числа ошибок. Подтверждено: {confirmed}.',
|
|
'', 'Диагностика: research.json содержит CPU/RAM каждого контейнера, активные ожидания PostgreSQL, счётчики ожидания пула и очереди/ошибки HAProxy. Pool-tuning — отдельный намеренно ограниченный сценарий; он не выдаётся за дефект исходного пула 8.',
|
|
'', 'Нужно интерпретировать насыщение по совместным данным: занятый пул и рост empty_acquires указывают на очередь пула; PostgreSQL около 100% при лимите 1 CPU и отсутствие прироста RPS указывают на CPU БД; очередь/CPU LB — на LB. Один плоский график не доказывает причину.',
|
|
'', 'Ограничения: один Docker Desktop/хост; closed-loop и coordinated omission; p95 только успехов; контейнерные CPU/RAM и PostgreSQL-счётчики включают прогрев; история растёт; дисковый кэш между сериями не сбрасывается; max RPS промышленной системы не заявляется.','']
|
|
(folder/'research.md').write_text('\n'.join(lines),encoding='utf-8')
|
|
return confirmed
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument('--duration',type=int,default=15)
|
|
p.add_argument('--warmup',type=int,default=3)
|
|
p.add_argument('--repeats',type=int,default=3)
|
|
p.add_argument('--resume',action='store_true',help='reuse completed runs with identical measurement parameters')
|
|
p.add_argument('--include-overload',action='store_true',help='also reproduce the rejected 64-client pool experiment')
|
|
args = p.parse_args()
|
|
if args.duration<10 or args.warmup<2 or args.repeats<3:
|
|
p.error('research needs duration>=10, warmup>=2, repeats>=3')
|
|
import matplotlib # fail before altering running services if dependencies are absent
|
|
folder = ROOT/'evidence'
|
|
folder.mkdir(exist_ok=True)
|
|
metadata = {'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'host':platform.platform(),
|
|
'host_cpu':platform.processor(),'logical_cpus':os.cpu_count(),
|
|
'docker':json.loads(docker('info','--format','{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"version":"{{.ServerVersion}}","os":"{{.OperatingSystem}}"}',capture=True)),
|
|
'profile':vars(args),'hypothesis':'Shared DB/LB or pool eventually limits scaling; more API replicas alone cannot remove that limit.'}
|
|
rows = []
|
|
saved = folder/'research.json'
|
|
if args.resume and saved.exists():
|
|
previous = json.loads(saved.read_text(encoding='utf-8'))
|
|
old_profile = previous['metadata']['profile']
|
|
if any(old_profile[k]!=getattr(args,k) for k in ('duration','warmup','repeats')):
|
|
p.error('saved measurement parameters differ; cannot resume')
|
|
metadata = previous['metadata']
|
|
rows = previous['runs']
|
|
# Upgrade early evidence where the pool telemetry shadowed its setting.
|
|
for row in rows:
|
|
if isinstance(row.get('pool'),dict):
|
|
row['pool_stats'] = row['pool']
|
|
row['pool'] = next(iter(row['pool_stats'].values()))['max_connections']
|
|
if row['series']=='pool-tuning' and row['concurrency']==64:
|
|
row['series']='pool-overload'
|
|
print('Reusing %d completed measurements' % len(rows),flush=True)
|
|
try:
|
|
for repeat in range(args.repeats):
|
|
configs = []
|
|
for series in ('fixed','growing'):
|
|
order = [1,2,4]
|
|
order = order[repeat%3:]+order[:repeat%3]
|
|
for n in order:
|
|
configs.append({'series':series,'replicas':n,'cpu_per_api':2/n if series=='fixed' else 1,
|
|
'memory_mib_per_api':1024//n if series=='fixed' else 512,
|
|
'pool':8//n if series=='fixed' else 8,'concurrency':32,'repeat':repeat+1})
|
|
for pool in ([1,8] if repeat%2==0 else [8,1]):
|
|
configs.append({'series':'pool-tuning','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':32,'repeat':repeat+1})
|
|
if args.include_overload:
|
|
for pool in ([1,8] if repeat%2==0 else [8,1]):
|
|
configs.append({'series':'pool-overload','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':64,'repeat':repeat+1})
|
|
for c in ([1,8,32,64,128] if repeat%2==0 else [128,64,32,8,1]):
|
|
configs.append({'series':'saturation','replicas':4,'cpu_per_api':1,'memory_mib_per_api':512,'pool':8,'concurrency':c,'repeat':repeat+1})
|
|
previous = None
|
|
for config in configs:
|
|
if any(all(row[k]==config[k] for k in ('series','replicas','concurrency','pool','repeat')) for row in rows):
|
|
continue
|
|
state = (config['replicas'],config['cpu_per_api'],config['memory_mib_per_api'],config['pool'])
|
|
if state!=previous:
|
|
configure(*state)
|
|
previous = state
|
|
print('Measuring '+json.dumps(config),flush=True)
|
|
rows.append(measure(config,args.duration,args.warmup))
|
|
(folder/'research.json').write_text(json.dumps({'metadata':metadata,'runs':rows},indent=2)+'\n',encoding='utf-8')
|
|
confirmed = summarize(rows,folder)
|
|
if not confirmed:
|
|
raise RuntimeError('pool tuning hypothesis was not confirmed; see research.md before claiming grade 5')
|
|
finally:
|
|
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api=2','api')
|
|
ready(2)
|
|
print('PASS: repeated scaling/saturation series and pool improvement measured; see evidence/research.md')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|