Выполнить лабораторную 3: балансировка, отказы и исследование масштабирования

This commit is contained in:
wpyatzy
2026-10-30 20:42:40 +03:00
parent 0ab3a341aa
commit 62de9484fe
112 changed files with 1187581 additions and 55 deletions
+108
View File
@@ -0,0 +1,108 @@
"""Persistent HTTP closed-loop benchmark, executed INSIDE the Compose network.
No retries: measures HTTP confirmations, with separate error/timeout counts.
Warmup is excluded. Individual latencies are retained, including failed requests.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import http.client
import json
import math
import os
from pathlib import Path
import sys
import threading
import time
import uuid
from urllib.parse import urlsplit
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
from client import Client
def benchmark(base, concurrency, duration, warmup):
c = Client(base)
alice, bob, eve, chat = c.bootstrap()
endpoint = '/api/v1/chats/' + chat['id'] + '/messages'
parsed = urlsplit(base)
barrier = threading.Barrier(concurrency + 1)
run_id = uuid.uuid4().hex
timing = {}
def worker(index):
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=5)
samples = []
observed = set()
counter = 0
barrier.wait()
while time.monotonic() < timing['end']:
start = time.monotonic()
measured = start >= timing['measure']
key = f'{run_id}-{index}-{counter}'
text = 'bench-' + key
status = 0
error = None
try:
conn.request('POST', endpoint, json.dumps({'text': text}), {
'Authorization': 'Bearer ' + alice['token'],
'Content-Type': 'application/json', 'Idempotency-Key': key})
response = conn.getresponse()
status = response.status
payload = response.read()
if status == 201:
data = json.loads(payload)
if data['text'] != text or data['chat_id'] != chat['id']:
error = 'invalid_payload'
if measured:
observed.add(response.getheader('X-Instance-Id'))
else:
error = 'http_' + str(status)
except (OSError, http.client.HTTPException, ValueError, KeyError) as exc:
error = type(exc).__name__
conn.close()
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=5)
elapsed = 1000 * (time.monotonic() - start)
if measured:
samples.append([round(elapsed, 3), status, error])
counter += 1
conn.close()
return samples, observed
with ThreadPoolExecutor(max_workers=concurrency) as pool:
futures = [pool.submit(worker, i) for i in range(concurrency)]
begin = time.monotonic()
timing.update(measure=begin + warmup, end=begin + warmup + duration)
barrier.wait()
results = [f.result() for f in futures]
end = time.monotonic()
samples = [s for values, _ in results for s in values]
successful = sorted(s[0] for s in samples if s[1] == 201 and s[2] is None)
errors = {}
for _, _, error in samples:
if error:
errors[error] = errors.get(error, 0) + 1
def percentile(p):
return successful[max(0, math.ceil(p * len(successful))-1)] if successful else None
return {'workload': 'closed-loop persistent POST via LB, unique keys, no retries',
'concurrency': concurrency, 'warmup_seconds': warmup,
'requested_duration_seconds': duration,
'measure_started_unix': time.time() - (time.monotonic() - timing['measure']),
'finished_unix': time.time(),
'elapsed_seconds': round(end - timing['measure'], 3),
'successful_requests': len(successful), 'errors': errors,
'successful_rps': round(len(successful) / (end-timing['measure']), 3),
'successful_latency_ms': {f'p{p}': percentile(p/100) for p in (50,95,99)},
'observed_instances': sorted({i for _, ids in results for i in ids if i}),
'samples_ms_status_error': samples}
if __name__ == '__main__':
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8083'))
parser.add_argument('--concurrency', type=int, default=16)
parser.add_argument('--duration', type=float, default=12)
parser.add_argument('--warmup', type=float, default=3)
args = parser.parse_args()
if not 1 <= args.concurrency <= 128 or not 1 <= args.duration <= 3600 or not 0 <= args.warmup <= 60:
parser.error('invalid load parameters')
print(json.dumps(benchmark(args.base_url, args.concurrency, args.duration, args.warmup)))
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
umask 077
python3 -c 'import secrets; print("POSTGRES_PASSWORD="+secrets.token_hex(24)); print("APP_PORT=8080")' > .env
fi
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if docker compose version >/dev/null 2>&1; then
exec docker compose "$@"
fi
exec docker-compose "$@"
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env python3
"""Sanitized real-container evidence. Never dump environment or labels wholesale."""
import json
from pathlib import Path
import subprocess
ROOT=Path(__file__).resolve().parents[1]
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-a','-q'],cwd=ROOT,text=True).split()
result=[]
for item in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)):
result.append({'name':item['Name'],'id':item['Id'],'hostname':item['Config']['Hostname'],
'service':item['Config']['Labels']['com.docker.compose.service'],
'image_id':item['Image'],'status':item['State']['Status'],
'started_at':item['State']['StartedAt'],
'health':item['State'].get('Health',{}).get('Status'),
'nano_cpus':item['HostConfig']['NanoCpus'],'memory_bytes':item['HostConfig']['Memory'],
'published_ports':item['HostConfig']['PortBindings'],
'volumes':[{'name':m['Name'],'destination':m['Destination']} for m in item['Mounts'] if m['Type']=='volume']})
print(json.dumps(sorted(result,key=lambda c:c['name']),indent=2))
+101
View File
@@ -0,0 +1,101 @@
#!/usr/bin/env python3
"""Generate standalone figures and CSV/Markdown tables from raw research JSON."""
from collections import defaultdict
import csv
import json
import os
import re
from pathlib import Path
import statistics
os.environ.setdefault('MPLCONFIGDIR', '/tmp/mtusi-lab3-matplotlib')
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
ROOT = Path(__file__).resolve().parents[1]
DIRECTORY = ROOT / 'evidence/scaling'
def memory_mib(value):
number, unit = re.fullmatch(r"([0-9.]+)([A-Za-z]+)", value.split(" / ")[0]).groups()
return float(number) * {"B":1/1048576,"KiB":1/1024,"MiB":1,"GiB":1024,"kB":1000/1048576,"MB":1000000/1048576,"GB":1000000000/1048576}[unit]
def main():
groups = defaultdict(list)
for path in DIRECTORY.glob('*.json'):
data = json.loads(path.read_text())
if 'case' in data:
groups[data['case']['name']].append(data)
rows = []
for name, values in sorted(groups.items()):
rps = [d['benchmark']['successful_rps'] for d in values]
row = {'case': name, 'repeats':len(values), 'replicas':values[0]['case']['replicas'],
'rps_mean':round(statistics.mean(rps),3), 'rps_min':min(rps), 'rps_max':max(rps),
'rps_stdev':round(statistics.stdev(rps),3) if len(rps)>1 else 0,
'p50_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p50'] for d in values),3),
'p95_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p95'] for d in values),3),
'p99_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p99'] for d in values),3),
'errors':sum(sum(d['benchmark']['errors'].values()) for d in values),
'db_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-db-1'] for d in values),3),
'lb_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-lb-1'] for d in values),3),
'client_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean'].get('mtusi-lab3-research-load',0) for d in values),3),
'api_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name'].startswith('mtusi-lab3-api-')),3) if any(r['Name'].startswith('mtusi-lab3-api-') for d in values for r in d['stats_samples']) else None,
'db_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name']=='mtusi-lab3-db-1'),3),
'cache_new_connections_mean':round(statistics.mean(d['dependency_after']['valkey']['total_connections_received']-d['dependency_before']['valkey']['total_connections_received'] for d in values),3)}
rows.append(row)
with (DIRECTORY/'summary.csv').open('w') as handle:
writer = csv.DictWriter(handle,fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
(DIRECTORY/'summary.json').write_text(json.dumps(rows,indent=2)+'\n')
by_name={row['case']:row for row in rows}
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.grid':True,'grid.alpha':.25})
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
for family,label in [('fixed','Fixed API: 1 CPU / 1024 MiB total'),('growing','Growing API: 0.5 CPU / 256 MiB each')]:
data=[by_name[f'{family}-{n}'] for n in (1,2,4) if f'{family}-{n}' in by_name]
if data:
for ax,metric in zip(axes,('rps_mean','p95_ms_mean')):
ax.plot([d['replicas'] for d in data],[d[metric] for d in data],marker='o',label=label)
axes[0].errorbar([d['replicas'] for d in data],[d['rps_mean'] for d in data],yerr=[d['rps_stdev'] for d in data],fmt='none',capsize=4)
for ax in axes:
ax.set_xlabel('API replicas')
ax.set_xticks([1,2,4])
axes[0].set_ylabel('Successful POST/s (mean ± sample SD)')
axes[1].set_ylabel('Successful request p95, ms (mean across runs)')
axes[0].legend(fontsize=8)
fig.suptitle('Replica scaling, concurrency 16; PostgreSQL limited to 0.25 CPU')
fig.savefig(DIRECTORY/'scaling.png',dpi=180)
fig.savefig(DIRECTORY/'scaling.svg')
plt.close(fig)
if all(f'saturation-c{c}' in by_name for c in (1,4,16,64)):
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
data=[by_name[f'saturation-c{c}'] for c in (1,4,16,64)]
axes[0].plot([1,4,16,64],[d['rps_mean'] for d in data],marker='o')
axes[0].set(xscale='log',xlabel='Concurrency',ylabel='Successful POST/s',title='Saturation at fixed resources')
ax2=axes[0].twinx()
ax2.plot([1,4,16,64],[d['p95_ms_mean'] for d in data],marker='s',color='tab:red')
ax2.set_ylabel('p95 ms',color='tab:red')
names=['saturation-c16','db-0.5','db-1']
if all(n in by_name for n in names):
axes[1].bar(['0.25 CPU','0.5 CPU','1 CPU'],[by_name[n]['rps_mean'] for n in names])
axes[1].set(xlabel='PostgreSQL CPU quota',ylabel='Successful POST/s',title='Bottleneck intervention (4 API, c=16)')
fig.savefig(DIRECTORY/'saturation.png',dpi=180)
fig.savefig(DIRECTORY/'saturation.svg')
plt.close(fig)
if all('cache-'+mode in by_name for mode in ('per_request','shared')):
fig,axes=plt.subplots(1,2,figsize=(10,4),layout='constrained')
names=['cache-per_request','cache-shared']
axes[0].bar(['Per-request client','Shared pool'],[by_name[n]['rps_mean'] for n in names], yerr=[by_name[n]['rps_stdev'] for n in names], capsize=5)
axes[0].set_ylabel('Successful POST/s')
axes[1].bar(['Per-request client','Shared pool'],[by_name[n]['cache_new_connections_mean'] for n in names])
axes[1].set_ylabel('New Valkey connections per full run')
fig.suptitle('Connection reuse: 2 API × 0.5 CPU, PostgreSQL 1 CPU, c=16')
fig.savefig(DIRECTORY/'cache-improvement.png',dpi=180)
fig.savefig(DIRECTORY/'cache-improvement.svg')
plt.close(fig)
print(json.dumps(rows,indent=2))
if __name__=='__main__':
main()
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""Reproducible lab3 track 1. Changes ONLY this Compose project's resources.
Restores the normal two-replica configuration in finally; does not delete volumes.
"""
import argparse
from collections import defaultdict
import json
import os
from pathlib import Path
import random
import subprocess
import threading
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = str(ROOT / 'scripts/compose.sh')
DEST = ROOT / 'evidence/scaling'
def call(*args, env=None):
return subprocess.check_output(args, cwd=ROOT, env=env, text=True).strip()
def compose(*args, env=None):
return call(COMPOSE, *args, env=env)
def inspect(ids):
return json.loads(call('docker', 'inspect', *ids))
def wait_ready(env, replicas, base):
deadline = time.monotonic()+75
while time.monotonic() < deadline:
ids = compose('ps', '-q', 'api', env=env).split()
if len(ids) == replicas:
state = inspect(ids)
if all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in state):
expected = {c['Config']['Hostname'] for c in state}
try:
seen = set()
for _ in range(32):
with urllib.request.urlopen(base+'/health/ready', timeout=2) as response:
seen.add(response.headers.get('X-Instance-Id'))
if seen == expected:
return
except OSError:
pass
time.sleep(.5)
raise RuntimeError('replicas/DNS/LB did not converge')
def dependency_stats(env):
sql = "SELECT json_build_object('xact_commit',xact_commit,'xact_rollback',xact_rollback,'blks_hit',blks_hit,'blks_read',blks_read,'tup_inserted',tup_inserted,'numbackends',numbackends) FROM pg_stat_database WHERE datname='messenger'"
db = json.loads(compose('exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',sql,env=env))
info = compose('exec','-T','valkey','valkey-cli','info','stats',env=env)
cache = {k: int(v.strip()) for line in info.splitlines() if ':' in line for k,v in [line.split(':',1)] if k in ('total_connections_received','total_commands_processed','keyspace_hits','keyspace_misses')}
return {'db':db,'valkey':cache}
class Stats:
def __init__(self, ids):
self.rows = []
self.proc = subprocess.Popen(['docker','stats','--format','{{json .}}','--no-trunc'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
self.thread = threading.Thread(target=self.read, daemon=True)
self.thread.start()
def read(self):
for line in self.proc.stdout:
try:
row = json.loads(line[line.find("{"):])
row['at_unix'] = time.time()
if row.get('Name', '').startswith('mtusi-lab3-'):
self.rows.append(row)
except ValueError:
pass
def close(self):
self.proc.kill()
self.proc.wait(timeout=5)
self.thread.join(timeout=5)
def run_case(case, index, args):
env = dict(os.environ, API_CPUS=str(case['api_cpu']), API_MEMORY=case['api_memory'],
API_REPLICAS=str(case['replicas']), DB_CPUS=str(case.get('db_cpu',.25)),
LB_CPUS=str(case.get('lb_cpu',.25)), DB_POOL_MAX_SIZE=str(case.get('pool',5)),
CACHE_CLIENT_MODE=case.get('cache','shared'),
COMPOSE_FILE=str(ROOT/'compose.yaml')+os.pathsep+str(ROOT/'compose.research.yaml'))
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api='+str(case['replicas']),'api','db','lb'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
wait_ready(env, case['replicas'], args.base_url)
ids = compose('ps','-q','api','db','lb','valkey',env=env).split()
configs = [{'name':c['Name'], 'id':c['Id'], 'image_id':c['Image'],
'cpu_limit':c['HostConfig']['NanoCpus']/1e9, 'memory_bytes':c['HostConfig']['Memory']}
for c in inspect(ids)]
before = dependency_stats(env)
observer = Stats(ids)
try:
output = compose('run','--rm','-T','--no-deps','--name','mtusi-lab3-research-load','load',
'--concurrency',str(case.get('concurrency',args.concurrency)),
'--duration',str(args.duration),'--warmup',str(args.warmup),env=env)
benchmark = json.loads(output)
finally:
observer.close()
after = dependency_stats(env)
cpu = defaultdict(list)
for row in observer.rows:
if benchmark['measure_started_unix'] <= row['at_unix'] <= benchmark['finished_unix']:
cpu[row['Name']].append(float(row['CPUPerc'].rstrip('%')))
report = {'case':case,'repeat':index,'benchmark':benchmark,'containers':configs,
'cpu_percent_mean':{k:round(sum(v)/len(v),3) for k,v in cpu.items()},
'stats_samples':observer.rows,'dependency_before':before,'dependency_after':after}
filename = case['name']+'-'+str(index)+'.json'
(DEST/filename).write_text(json.dumps(report, indent=2)+'\n')
print(json.dumps({'file':filename,'rps':benchmark['successful_rps'],
'p95_ms':benchmark['successful_latency_ms']['p95'],
'errors':benchmark['errors'],'cpu':report['cpu_percent_mean']}), flush=True)
return filename
def main():
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url',default=os.getenv('BASE_URL','http://localhost:8083'))
parser.add_argument('--duration',type=float,default=12)
parser.add_argument('--warmup',type=float,default=3)
parser.add_argument('--repeats',type=int,default=3)
parser.add_argument('--concurrency',type=int,default=16)
parser.add_argument('--suite',choices=['scaling','saturation','improvement','all'],default='all')
args=parser.parse_args()
DEST.mkdir(parents=True,exist_ok=True)
cases=[]
if args.suite in ('scaling','all'):
for n in (1,2,4):
cases.append({'name':f'fixed-{n}','replicas':n,'api_cpu':1/n,'api_memory':str(1024//n)+'m','family':'fixed'})
cases.append({'name':f'growing-{n}','replicas':n,'api_cpu':.5,'api_memory':'256m','family':'growing'})
if args.suite in ('saturation','all'):
for concurrency in (1,4,16,64):
cases.append({'name':f'saturation-c{concurrency}','replicas':4,'api_cpu':.5,'api_memory':'256m','concurrency':concurrency,'family':'saturation'})
for db_cpu in (.5,1):
cases.append({'name':f'db-{db_cpu}','replicas':4,'api_cpu':.5,'api_memory':'256m','db_cpu':db_cpu,'family':'db-quota'})
if args.suite in ('improvement','all'):
for cache in ('per_request','shared'):
cases.append({'name':'cache-'+cache,'replicas':2,'api_cpu':.5,'api_memory':'256m','db_cpu':1,'cache':cache,'family':'cache'})
files=[]
try:
for repeat in range(1,args.repeats+1):
ordered=list(cases)
random.Random(20261030+repeat).shuffle(ordered)
for case in ordered:
files.append(run_case(case,repeat,args))
finally:
# Restore normal config from .env, not the last experimental overrides.
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api=2','api','db','lb'],cwd=ROOT,check=True)
(DEST/('manifest-'+args.suite+'.json')).write_text(json.dumps({'suite':args.suite,'parameters':vars(args),'files':files,'seed_base':20261030},indent=2)+'\n')
if __name__=='__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
scripts/compose.sh restart db valkey api lb
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/rolling_update.py
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Replace each API container using a healthy temporary extra replica (surge=1)."""
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = str(ROOT / 'scripts/compose.sh')
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def ids():
return command(COMPOSE, 'ps', '-q', 'api').split()
def inspect(cid):
result = json.loads(command('docker', 'inspect', cid))[0]
return {'id': result['Id'], 'hostname': result['Config']['Hostname'],
'image_id': result['Image'], 'image': result['Config']['Image'],
'status': result['State']['Status'],
'health': result['State'].get('Health', {}).get('Status')}
def main():
original = ids()
if len(original) < 2:
raise RuntimeError('rolling update requires at least two running replicas')
base = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
evidence = {'before': [inspect(i) for i in original], 'steps': []}
try:
for old in original:
before = set(ids())
subprocess.run([COMPOSE, 'up', '-d', '--no-deps', '--no-recreate',
'--scale', 'api=' + str(len(original)+1), 'api'], check=True, cwd=ROOT)
added = set(ids()) - before
if len(added) != 1:
raise RuntimeError('expected exactly one spare container')
spare = added.pop()
deadline = time.monotonic() + 60
while True:
info = inspect(spare)
# Healthy container must actually be observed behind the public LB.
seen = False
if info['health'] == 'healthy':
try:
for _ in range(2 * (len(original) + 1)):
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
if response.headers.get('X-Instance-Id') == info['hostname']:
seen = True
break
except (OSError, urllib.error.URLError):
pass
if seen:
break
if time.monotonic() >= deadline:
raise RuntimeError('spare did not become healthy and reachable via LB')
time.sleep(.3)
subprocess.run(['docker', 'stop', '--time', '12', old], check=True, stdout=subprocess.DEVNULL)
stopped = inspect(old)
subprocess.run(['docker', 'rm', old], check=True, stdout=subprocess.DEVNULL)
evidence['steps'].append({'removed': stopped, 'replacement': info})
evidence['after'] = [inspect(i) for i in ids()]
assert len(evidence['after']) == len(original)
assert not set(original) & set(ids())
evidence['status'] = 'PASS'
finally:
dest = ROOT / 'evidence/rolling-containers.json'
dest.write_text(json.dumps(evidence, indent=2) + '\n')
print('PASS: all original API containers replaced; evidence/rolling-containers.json')
if __name__ == '__main__':
main()
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?Required API hostname from X-Instance-Id}"
for id in $(scripts/compose.sh ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 3 "$id" >/dev/null
exit 0
fi
done
printf '%s\n' 'Target is not a running API replica' >&2
exit 1
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""Append tables and evidence-based conclusions to the research methodology."""
import json
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
rows=json.loads((ROOT/'evidence/scaling/summary.json').read_text())
by_name={r['case']:r for r in rows}
report=ROOT/'RESEARCH.md'
text=report.read_text().split('\n## Результаты\n')[0]
text+='\n## Результаты\n\n'
text+='Каждая строка — три повтора. Ошибки во всех 42 запусках отсутствовали. RPS±s — среднее и выборочное стандартное отклонение.\n\n'
text+='| Серия | Реплик | RPS ± s | min–max RPS | p50 / p95 / p99, мс | CPU БД, % одного CPU |\n| --- | --- | --- | --- | --- | --- |\n'
for family in ('fixed','growing'):
for n in (1,2,4):
r=by_name[f'{family}-{n}']
text+=f"| {family} | {n} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['rps_min']:.1f}–{r['rps_max']:.1f} | {r['p50_ms_mean']:.1f} / {r['p95_ms_mean']:.1f} / {r['p99_ms_mean']:.1f} | {r['db_cpu_percent_mean']:.2f} |\n"
text+='\n![Масштабирование API](evidence/scaling/scaling.png)\n\n'
text+='При фиксированном бюджете 1→4 реплики снизили throughput, а p95 вырос. Это опровергает ожидание «реплик больше — всегда быстрее». Разделение бюджета между процессами, независимые пулы/healthchecks, синхронный I/O и планирование дробных CPU-квот могут менять эффективность. Вклад каждого фактора здесь отдельно не измерялся, поэтому конкретная причина всей деградации fixed не доказана. При росте бюджета переход 2→4 также не дал устойчивого выигрыша: БД работала около своей полной квоты.\n\n'
text+='### Насыщение и вмешательство в ресурс БД\n\n'
text+='| Конфигурация: 4 API × 0.5 CPU | RPS ± s | p95, мс | CPU БД, % одного CPU | CPU LB / клиента, % |\n| --- | --- | --- | --- | --- |\n'
for name in ('saturation-c1','saturation-c4','saturation-c16','saturation-c64','db-0.5','db-1'):
r=by_name[name]
text+=f"| {name} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['p95_ms_mean']:.1f} | {r['db_cpu_percent_mean']:.2f} | {r['lb_cpu_percent_mean']:.2f} / {r['client_cpu_percent_mean']:.2f} |\n"
a,b=by_name['saturation-c16'],by_name['db-1']
text+=f"\nПри c=4 БД уже близка к квоте 25%; максимум среднего throughput среди исследованных c находится при c=16 ({a['rps_mean']:.1f} POST/с). Рост c до 64 не дал прироста, p95 вырос до {by_name['saturation-c64']['p95_ms_mean']:.1f} мс. Это эмпирическая область насыщения около 400–500 POST/с для данной конфигурации, не точная универсальная граница. При увеличении квоты только БД до 1 CPU средний throughput вырос в {b['rps_mean']/a['rps_mean']:.2f} раза, до {b['rps_mean']:.1f} POST/с. Это подтверждает ограничение общей зависимости в исходном стенде.\n\n"
text+='LB и генератор существенно ниже своих CPU-лимитов 25% и 100% соответственно; их CPU не объясняет исходное плато. При DB=1 CPU сама БД уже не достигает квоты; ограничение сдвигается. Точное новое узкое место (Python/I/O/планировщик/пул) этим экспериментом не локализовано. Один процесс API выполняет синхронные операции БД в async-обработчике, поэтому добавление размера пула не гарантирует параллелизм. Не заявляется, что достигнута предельная мощность PostgreSQL или устранены все bottleneck.\n\n'
text+='![Насыщение и CPU БД](evidence/scaling/saturation.png)\n\n'
text+='### Подтверждённое улучшение: переиспользование соединений Valkey\n\n'
text+='Объект Redis теперь создаётся один раз на Store/процесс и использует библиотечный connection pool; закрывается при shutdown. Для контрольного эксперимента CACHE_CLIENT_MODE=per_request воспроизводит прежнее создание клиента на каждую операцию. По умолчанию выбран shared. Проверка идентичности/expiry/revocation в PostgreSQL не меняется.\n\n'
text+='| Вариант | RPS ± s | min–max RPS | p95, мс | Новых соединений Valkey за весь запуск, среднее |\n| --- | --- | --- | --- | --- |\n'
for name in ('cache-per_request','cache-shared'):
r=by_name[name]
text+=f"| {name} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['rps_min']:.1f}–{r['rps_max']:.1f} | {r['p95_ms_mean']:.1f} | {r['cache_new_connections_mean']:.3f} |\n"
a,b=by_name['cache-per_request'],by_name['cache-shared']
text+=f"\nВ этих трёх повторах средний throughput вырос на {100*(b['rps_mean']/a['rps_mean']-1):.1f}%, средний p95 снизился на {100*(1-b['p95_ms_mean']/a['p95_ms_mean']):.1f}%. Число новых соединений снизилось на {100*(1-b['cache_new_connections_mean']/a['cache_new_connections_mean']):.3f}%. Интервалы min–max throughput перекрываются, статистическая значимость не проверялась; гарантировать прирост 31% для иной нагрузки нельзя. Устойчиво подтверждено устранение почти всех повторных подключений.\n\n"
text+='![Переиспользование соединений](evidence/scaling/cache-improvement.png)\n\n'
text+='## Ограничения и практический вывод\n\n'
text+='Три коротких повтора по десять секунд, короткий прогрев, растущие данные и VM дают заметный разброс. Не контролировались page cache, фазы CFS-квот, частота CPU и фоновая активность ОС. Стандартное отклонение не является доверительным интервалом. Измеряется одна короткая POST-нагрузка с действующими сессиями, а не смесь логинов, чтения больших историй и файлов. Нет open-loop arrival rate, длинного soak-теста, статистического теста значимости или доказательства исчерпания RAM. Raw summaries содержат memory usage/limits, а не профили heap.\n\n'
text+='До увеличения числа API нужно измерять общую зависимость: в этом стенде DB CPU объясняет плато growing, а дробление фиксированного CPU не помогает. Подтверждены два практических эффекта: снятие искусственно тесной квоты БД увеличивает пропускную способность; общий пул Valkey устраняет churn соединений и улучшает средние показатели этого профиля. При этом в обычной конфигурации сохранены две API для отказоустойчивости процесса, а не выбрана единственная реплика ради лучшего числа в fixed-серии.\n'
report.write_text(text)