"""Grade-5 track 1: controlled scaling, saturation sweep and pool tuning.""" import argparse import csv import datetime as dt import io import json import os from pathlib import Path import platform import statistics import subprocess import threading import time from manage import ROOT, docker, ready def sql(query): return json.loads(docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-At','-c',query,capture=True)) def db_stats(): return sql("SELECT row_to_json(s) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_inserted,deadlocks FROM pg_stat_database WHERE datname=current_database()) s") def lb_stats(): text = docker('compose','exec','-T','lb','wget','-q','-O','-','http://127.0.0.1:8404/stats;csv',capture=True) rows = list(csv.DictReader(io.StringIO(text.removeprefix('# ')))) return [{k:r.get(k) for k in ('svname','status','qcur','qmax','scur','smax','econ','eresp','stot')} for r in rows if r.get('pxname')=='api'] def configure(n, cpu, memory, pool): env = dict(os.environ, DB_CPUS='1.0', API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=str(memory)+'m', DB_POOL_MAX=str(pool), POOL_METRICS='1', REQUEST_LOG='0') docker('compose','up','-d','--no-deps','--force-recreate','--scale','api='+str(n),'api',env=env) ready(n) return env def measure(config, duration, warmup): began = dt.datetime.now(dt.timezone.utc).isoformat() before, lb_before = db_stats(), lb_stats() samples = [] finished = threading.Event() sample_errors = [] def monitor(): while not finished.is_set(): try: ids = docker('ps','-q','--filter','label=com.docker.compose.project=mtusi-lab3',capture=True).split() ids += docker('ps','-q','--filter','label=mtusi.lab3.role=load',capture=True).split() text = docker('stats','--no-stream','--format','{{json .}}',*ids,capture=True) stats = [json.loads(line) for line in text.splitlines() if line.strip()] waits = sql("SELECT coalesce(json_object_agg(kind,n),'{}') FROM (SELECT coalesce(wait_event_type,'CPU/runnable') kind,count(*) n FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active' GROUP BY 1) s") samples.append({'time':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':stats,'db_active_waits':waits}) except (subprocess.CalledProcessError, ValueError) as error: sample_errors.append(type(error).__name__) finished.wait(.5) thread = threading.Thread(target=monitor) thread.start() try: output = docker('run','--rm','--network','mtusi-lab3_default','--cpus','2','--memory','256m', '--label','mtusi.lab3.role=load','mtusi-messenger:lab3-load', '-duration',str(duration)+'s','-warmup',str(warmup)+'s', '-concurrency',str(config['concurrency']),capture=True) result = json.loads(output) finally: finished.set() thread.join() after = db_stats() logs = docker('compose','logs','--no-log-prefix','--since',began,'api',capture=True) pools = {} for line in logs.splitlines(): try: item = json.loads(line) except ValueError: continue if item.get('msg')=='pool': pools.setdefault(item['instance'],[]).append(item) pool_summary = {} for instance, values in pools.items(): pool_summary[instance] = {'max_acquired':max(v['acquired'] for v in values),'max_connections':values[-1]['max'], 'empty_acquires_delta':values[-1]['empty_acquires']-values[0]['empty_acquires'], 'acquire_seconds_delta':(values[-1]['acquire_ns']-values[0]['acquire_ns'])/1e9} if len(result['instances']) != config['replicas']: raise RuntimeError('load did not reach the configured number of replicas') if not samples or not pool_summary: raise RuntimeError('missing CPU/DB/pool observations') return dict(config, result=result, samples=samples, sample_errors=sample_errors, pool_stats=pool_summary, db_delta={k:after[k]-before[k] for k in before},lb_before=lb_before,lb_after=lb_stats()) def summarize(rows, folder): import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt grouped = {} for row in rows: key = (row['series'],row['replicas'],row['concurrency'],row['pool']) grouped.setdefault(key,[]).append(row) summaries = [] for key, values in sorted(grouped.items()): rps = [r['result']['successful_rps'] for r in values] summaries.append({'series':key[0],'replicas':key[1],'concurrency':key[2],'pool':key[3], 'rps_median':statistics.median(rps),'rps_min':min(rps),'rps_max':max(rps), 'p95_ms_median':statistics.median(r['result']['latency_ms']['p95'] for r in values), 'failed_requests':sum(r['result']['failed_requests'] for r in values), 'successful_requests':sum(r['result']['successful_requests'] for r in values)}) with (folder/'research.csv').open('w',newline='',encoding='utf-8') as output: writer = csv.DictWriter(output,fieldnames=list(summaries[0])) writer.writeheader() writer.writerows(summaries) fig, axes = plt.subplots(1,2,figsize=(11,4),layout='constrained') labels = {'fixed':'fixed total: 2 CPU / 1024 MiB','growing':'growing: 1 CPU / 512 MiB per API'} for series in ('fixed','growing'): points = sorted((s for s in summaries if s['series']==series),key=lambda p:p['replicas']) axes[0].errorbar([p['replicas'] for p in points],[p['rps_median'] for p in points], yerr=[[p['rps_median']-p['rps_min'] for p in points],[p['rps_max']-p['rps_median'] for p in points]],marker='o',capsize=4,label=labels[series]) axes[1].plot([p['replicas'] for p in points],[p['p95_ms_median'] for p in points],marker='o',label=labels[series]) for axis in axes: axis.set_xlabel('API replicas') axis.set_xticks([1,2,4]) axis.grid(alpha=.25) axis.legend(fontsize=8) axis.set_ylim(bottom=0) fig.suptitle('3 repetitions; 15 s after 3 s warm-up; 32 closed-loop clients') axes[0].set_ylabel('Successful HTTP confirmations / second') axes[0].set_title('Median and range across repetitions') axes[1].set_ylabel('p95 of successful requests, ms') fig.savefig(folder/'scaling.png',dpi=160) plt.close(fig) sweep = sorted((s for s in summaries if s['series']=='saturation'),key=lambda p:p['concurrency']) fig, axes = plt.subplots(1,3,figsize=(12,4),layout='constrained') clients = [p['concurrency'] for p in sweep] axes[0].errorbar(clients,[p['rps_median'] for p in sweep], yerr=[[p['rps_median']-p['rps_min'] for p in sweep],[p['rps_max']-p['rps_median'] for p in sweep]],marker='o',capsize=4) axes[0].set_ylabel('Successful confirmations / second') axes[1].plot(clients,[p['p95_ms_median'] for p in sweep],marker='o') axes[1].set_ylabel('p95 of successful requests, ms') axes[2].plot(clients,[100*p['failed_requests']/(p['failed_requests']+p['successful_requests']) for p in sweep],marker='o',color='tab:red') axes[2].set_ylabel('Pooled failed attempts, %') for axis in axes: axis.set_xscale('log',base=2) axis.set_xticks(clients,labels=[str(c) for c in clients]) axis.set_xlabel('Concurrent clients') axis.set_ylim(bottom=0) axis.grid(alpha=.25) fig.suptitle('Saturation: 4 API replicas; 3 repetitions; 15 s measurements') fig.savefig(folder/'saturation.png',dpi=160) plt.close(fig) tuning = {s['pool']:s for s in summaries if s['series']=='pool-tuning' and s['concurrency']==32} if set(tuning)!={1,8}: raise RuntimeError('controlled pool tuning at concurrency 32 is missing; continue with --resume') improvement = tuning[8]['rps_median']/tuning[1]['rps_median']-1 confirmed = improvement>.05 and tuning[8]['failed_requests']<=tuning[1]['failed_requests'] lines = ['# Измерения трека 1','', 'Fixed: суммарно 2 CPU / 1024 MiB API, суммарный пул 8. Growing: 1 CPU / 512 MiB и пул 8 на каждую API. PostgreSQL 1 CPU / 512 MiB, LB 0.5 CPU / 128 MiB, Valkey 0.5 CPU / 256 MiB, генератор 2 CPU / 256 MiB.','', '| Серия | API | Клиенты | Пул/реплика | RPS median [min;max] | p95 ms | Ошибки |', '|---|---:|---:|---:|---:|---:|---:|'] for s in summaries: lines.append('| {series} | {replicas} | {concurrency} | {pool} | {rps_median:.1f} [{rps_min:.1f}; {rps_max:.1f}] | {p95_ms_median:.1f} | {failed_requests} |'.format(**s)) lines += ['',f'Контролируемая настройка пула 1 → 8 (2 API, 32 клиента): изменение median RPS {improvement:+.1%}. Критерий подтверждения: >5% без увеличения общего числа ошибок. Подтверждено: {confirmed}.', '', 'Диагностика: research.json содержит CPU/RAM каждого контейнера, активные ожидания PostgreSQL, счётчики ожидания пула и очереди/ошибки HAProxy. Pool-tuning — отдельный намеренно ограниченный сценарий; он не выдаётся за дефект исходного пула 8.', '', 'Нужно интерпретировать насыщение по совместным данным: занятый пул и рост empty_acquires указывают на очередь пула; PostgreSQL около 100% при лимите 1 CPU и отсутствие прироста RPS указывают на CPU БД; очередь/CPU LB — на LB. Один плоский график не доказывает причину.', '', 'Ограничения: один Docker Desktop/хост; closed-loop и coordinated omission; p95 только успехов; контейнерные CPU/RAM и PostgreSQL-счётчики включают прогрев; история растёт; дисковый кэш между сериями не сбрасывается; max RPS промышленной системы не заявляется.',''] (folder/'research.md').write_text('\n'.join(lines),encoding='utf-8') return confirmed def main(): p = argparse.ArgumentParser(description=__doc__) p.add_argument('--duration',type=int,default=15) p.add_argument('--warmup',type=int,default=3) p.add_argument('--repeats',type=int,default=3) p.add_argument('--resume',action='store_true',help='reuse completed runs with identical measurement parameters') p.add_argument('--include-overload',action='store_true',help='also reproduce the rejected 64-client pool experiment') args = p.parse_args() if args.duration<10 or args.warmup<2 or args.repeats<3: p.error('research needs duration>=10, warmup>=2, repeats>=3') import matplotlib # fail before altering running services if dependencies are absent folder = ROOT/'evidence' folder.mkdir(exist_ok=True) metadata = {'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'host':platform.platform(), 'host_cpu':platform.processor(),'logical_cpus':os.cpu_count(), 'docker':json.loads(docker('info','--format','{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"version":"{{.ServerVersion}}","os":"{{.OperatingSystem}}"}',capture=True)), 'profile':vars(args),'hypothesis':'Shared DB/LB or pool eventually limits scaling; more API replicas alone cannot remove that limit.'} rows = [] saved = folder/'research.json' if args.resume and saved.exists(): previous = json.loads(saved.read_text(encoding='utf-8')) old_profile = previous['metadata']['profile'] if any(old_profile[k]!=getattr(args,k) for k in ('duration','warmup','repeats')): p.error('saved measurement parameters differ; cannot resume') metadata = previous['metadata'] rows = previous['runs'] # Upgrade early evidence where the pool telemetry shadowed its setting. for row in rows: if isinstance(row.get('pool'),dict): row['pool_stats'] = row['pool'] row['pool'] = next(iter(row['pool_stats'].values()))['max_connections'] if row['series']=='pool-tuning' and row['concurrency']==64: row['series']='pool-overload' print('Reusing %d completed measurements' % len(rows),flush=True) try: for repeat in range(args.repeats): configs = [] for series in ('fixed','growing'): order = [1,2,4] order = order[repeat%3:]+order[:repeat%3] for n in order: configs.append({'series':series,'replicas':n,'cpu_per_api':2/n if series=='fixed' else 1, 'memory_mib_per_api':1024//n if series=='fixed' else 512, 'pool':8//n if series=='fixed' else 8,'concurrency':32,'repeat':repeat+1}) for pool in ([1,8] if repeat%2==0 else [8,1]): configs.append({'series':'pool-tuning','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':32,'repeat':repeat+1}) if args.include_overload: for pool in ([1,8] if repeat%2==0 else [8,1]): configs.append({'series':'pool-overload','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':64,'repeat':repeat+1}) for c in ([1,8,32,64,128] if repeat%2==0 else [128,64,32,8,1]): configs.append({'series':'saturation','replicas':4,'cpu_per_api':1,'memory_mib_per_api':512,'pool':8,'concurrency':c,'repeat':repeat+1}) previous = None for config in configs: if any(all(row[k]==config[k] for k in ('series','replicas','concurrency','pool','repeat')) for row in rows): continue state = (config['replicas'],config['cpu_per_api'],config['memory_mib_per_api'],config['pool']) if state!=previous: configure(*state) previous = state print('Measuring '+json.dumps(config),flush=True) rows.append(measure(config,args.duration,args.warmup)) (folder/'research.json').write_text(json.dumps({'metadata':metadata,'runs':rows},indent=2)+'\n',encoding='utf-8') confirmed = summarize(rows,folder) if not confirmed: raise RuntimeError('pool tuning hypothesis was not confirmed; see research.md before claiming grade 5') finally: docker('compose','up','-d','--no-deps','--force-recreate','--scale','api=2','api') ready(2) print('PASS: repeated scaling/saturation series and pool improvement measured; see evidence/research.md') if __name__=='__main__': main()