forked from mtusi2026/mtusi2026lab3
Выполнить лабораторную 3: балансировка, отказы и исследование масштабирования
This commit is contained in:
@@ -0,0 +1,108 @@
|
||||
"""Persistent HTTP closed-loop benchmark, executed INSIDE the Compose network.
|
||||
|
||||
No retries: measures HTTP confirmations, with separate error/timeout counts.
|
||||
Warmup is excluded. Individual latencies are retained, including failed requests.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client
|
||||
|
||||
|
||||
def benchmark(base, concurrency, duration, warmup):
|
||||
c = Client(base)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
endpoint = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
parsed = urlsplit(base)
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
run_id = uuid.uuid4().hex
|
||||
timing = {}
|
||||
|
||||
def worker(index):
|
||||
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=5)
|
||||
samples = []
|
||||
observed = set()
|
||||
counter = 0
|
||||
barrier.wait()
|
||||
while time.monotonic() < timing['end']:
|
||||
start = time.monotonic()
|
||||
measured = start >= timing['measure']
|
||||
key = f'{run_id}-{index}-{counter}'
|
||||
text = 'bench-' + key
|
||||
status = 0
|
||||
error = None
|
||||
try:
|
||||
conn.request('POST', endpoint, json.dumps({'text': text}), {
|
||||
'Authorization': 'Bearer ' + alice['token'],
|
||||
'Content-Type': 'application/json', 'Idempotency-Key': key})
|
||||
response = conn.getresponse()
|
||||
status = response.status
|
||||
payload = response.read()
|
||||
if status == 201:
|
||||
data = json.loads(payload)
|
||||
if data['text'] != text or data['chat_id'] != chat['id']:
|
||||
error = 'invalid_payload'
|
||||
if measured:
|
||||
observed.add(response.getheader('X-Instance-Id'))
|
||||
else:
|
||||
error = 'http_' + str(status)
|
||||
except (OSError, http.client.HTTPException, ValueError, KeyError) as exc:
|
||||
error = type(exc).__name__
|
||||
conn.close()
|
||||
conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=5)
|
||||
elapsed = 1000 * (time.monotonic() - start)
|
||||
if measured:
|
||||
samples.append([round(elapsed, 3), status, error])
|
||||
counter += 1
|
||||
conn.close()
|
||||
return samples, observed
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
futures = [pool.submit(worker, i) for i in range(concurrency)]
|
||||
begin = time.monotonic()
|
||||
timing.update(measure=begin + warmup, end=begin + warmup + duration)
|
||||
barrier.wait()
|
||||
results = [f.result() for f in futures]
|
||||
end = time.monotonic()
|
||||
samples = [s for values, _ in results for s in values]
|
||||
successful = sorted(s[0] for s in samples if s[1] == 201 and s[2] is None)
|
||||
errors = {}
|
||||
for _, _, error in samples:
|
||||
if error:
|
||||
errors[error] = errors.get(error, 0) + 1
|
||||
def percentile(p):
|
||||
return successful[max(0, math.ceil(p * len(successful))-1)] if successful else None
|
||||
return {'workload': 'closed-loop persistent POST via LB, unique keys, no retries',
|
||||
'concurrency': concurrency, 'warmup_seconds': warmup,
|
||||
'requested_duration_seconds': duration,
|
||||
'measure_started_unix': time.time() - (time.monotonic() - timing['measure']),
|
||||
'finished_unix': time.time(),
|
||||
'elapsed_seconds': round(end - timing['measure'], 3),
|
||||
'successful_requests': len(successful), 'errors': errors,
|
||||
'successful_rps': round(len(successful) / (end-timing['measure']), 3),
|
||||
'successful_latency_ms': {f'p{p}': percentile(p/100) for p in (50,95,99)},
|
||||
'observed_instances': sorted({i for _, ids in results for i in ids if i}),
|
||||
'samples_ms_status_error': samples}
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.getenv('BASE_URL', 'http://localhost:8083'))
|
||||
parser.add_argument('--concurrency', type=int, default=16)
|
||||
parser.add_argument('--duration', type=float, default=12)
|
||||
parser.add_argument('--warmup', type=float, default=3)
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 128 or not 1 <= args.duration <= 3600 or not 0 <= args.warmup <= 60:
|
||||
parser.error('invalid load parameters')
|
||||
print(json.dumps(benchmark(args.base_url, args.concurrency, args.duration, args.warmup)))
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ ! -f .env ]; then
|
||||
umask 077
|
||||
python3 -c 'import secrets; print("POSTGRES_PASSWORD="+secrets.token_hex(24)); print("APP_PORT=8080")' > .env
|
||||
fi
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if docker compose version >/dev/null 2>&1; then
|
||||
exec docker compose "$@"
|
||||
fi
|
||||
exec docker-compose "$@"
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sanitized real-container evidence. Never dump environment or labels wholesale."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
ids=subprocess.check_output([str(ROOT/'scripts/compose.sh'),'ps','-a','-q'],cwd=ROOT,text=True).split()
|
||||
result=[]
|
||||
for item in json.loads(subprocess.check_output(['docker','inspect',*ids],text=True)):
|
||||
result.append({'name':item['Name'],'id':item['Id'],'hostname':item['Config']['Hostname'],
|
||||
'service':item['Config']['Labels']['com.docker.compose.service'],
|
||||
'image_id':item['Image'],'status':item['State']['Status'],
|
||||
'started_at':item['State']['StartedAt'],
|
||||
'health':item['State'].get('Health',{}).get('Status'),
|
||||
'nano_cpus':item['HostConfig']['NanoCpus'],'memory_bytes':item['HostConfig']['Memory'],
|
||||
'published_ports':item['HostConfig']['PortBindings'],
|
||||
'volumes':[{'name':m['Name'],'destination':m['Destination']} for m in item['Mounts'] if m['Type']=='volume']})
|
||||
print(json.dumps(sorted(result,key=lambda c:c['name']),indent=2))
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate standalone figures and CSV/Markdown tables from raw research JSON."""
|
||||
from collections import defaultdict
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
|
||||
os.environ.setdefault('MPLCONFIGDIR', '/tmp/mtusi-lab3-matplotlib')
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIRECTORY = ROOT / 'evidence/scaling'
|
||||
|
||||
|
||||
def memory_mib(value):
|
||||
number, unit = re.fullmatch(r"([0-9.]+)([A-Za-z]+)", value.split(" / ")[0]).groups()
|
||||
return float(number) * {"B":1/1048576,"KiB":1/1024,"MiB":1,"GiB":1024,"kB":1000/1048576,"MB":1000000/1048576,"GB":1000000000/1048576}[unit]
|
||||
|
||||
|
||||
def main():
|
||||
groups = defaultdict(list)
|
||||
for path in DIRECTORY.glob('*.json'):
|
||||
data = json.loads(path.read_text())
|
||||
if 'case' in data:
|
||||
groups[data['case']['name']].append(data)
|
||||
rows = []
|
||||
for name, values in sorted(groups.items()):
|
||||
rps = [d['benchmark']['successful_rps'] for d in values]
|
||||
row = {'case': name, 'repeats':len(values), 'replicas':values[0]['case']['replicas'],
|
||||
'rps_mean':round(statistics.mean(rps),3), 'rps_min':min(rps), 'rps_max':max(rps),
|
||||
'rps_stdev':round(statistics.stdev(rps),3) if len(rps)>1 else 0,
|
||||
'p50_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p50'] for d in values),3),
|
||||
'p95_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p95'] for d in values),3),
|
||||
'p99_ms_mean':round(statistics.mean(d['benchmark']['successful_latency_ms']['p99'] for d in values),3),
|
||||
'errors':sum(sum(d['benchmark']['errors'].values()) for d in values),
|
||||
'db_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-db-1'] for d in values),3),
|
||||
'lb_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean']['mtusi-lab3-lb-1'] for d in values),3),
|
||||
'client_cpu_percent_mean':round(statistics.mean(d['cpu_percent_mean'].get('mtusi-lab3-research-load',0) for d in values),3),
|
||||
'api_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name'].startswith('mtusi-lab3-api-')),3) if any(r['Name'].startswith('mtusi-lab3-api-') for d in values for r in d['stats_samples']) else None,
|
||||
'db_memory_peak_mib':round(max(memory_mib(r['MemUsage']) for d in values for r in d['stats_samples'] if r['Name']=='mtusi-lab3-db-1'),3),
|
||||
'cache_new_connections_mean':round(statistics.mean(d['dependency_after']['valkey']['total_connections_received']-d['dependency_before']['valkey']['total_connections_received'] for d in values),3)}
|
||||
rows.append(row)
|
||||
with (DIRECTORY/'summary.csv').open('w') as handle:
|
||||
writer = csv.DictWriter(handle,fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
(DIRECTORY/'summary.json').write_text(json.dumps(rows,indent=2)+'\n')
|
||||
by_name={row['case']:row for row in rows}
|
||||
plt.rcParams.update({'font.family':'DejaVu Sans','font.size':10,'axes.grid':True,'grid.alpha':.25})
|
||||
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
||||
for family,label in [('fixed','Fixed API: 1 CPU / 1024 MiB total'),('growing','Growing API: 0.5 CPU / 256 MiB each')]:
|
||||
data=[by_name[f'{family}-{n}'] for n in (1,2,4) if f'{family}-{n}' in by_name]
|
||||
if data:
|
||||
for ax,metric in zip(axes,('rps_mean','p95_ms_mean')):
|
||||
ax.plot([d['replicas'] for d in data],[d[metric] for d in data],marker='o',label=label)
|
||||
axes[0].errorbar([d['replicas'] for d in data],[d['rps_mean'] for d in data],yerr=[d['rps_stdev'] for d in data],fmt='none',capsize=4)
|
||||
for ax in axes:
|
||||
ax.set_xlabel('API replicas')
|
||||
ax.set_xticks([1,2,4])
|
||||
axes[0].set_ylabel('Successful POST/s (mean ± sample SD)')
|
||||
axes[1].set_ylabel('Successful request p95, ms (mean across runs)')
|
||||
axes[0].legend(fontsize=8)
|
||||
fig.suptitle('Replica scaling, concurrency 16; PostgreSQL limited to 0.25 CPU')
|
||||
fig.savefig(DIRECTORY/'scaling.png',dpi=180)
|
||||
fig.savefig(DIRECTORY/'scaling.svg')
|
||||
plt.close(fig)
|
||||
if all(f'saturation-c{c}' in by_name for c in (1,4,16,64)):
|
||||
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
||||
data=[by_name[f'saturation-c{c}'] for c in (1,4,16,64)]
|
||||
axes[0].plot([1,4,16,64],[d['rps_mean'] for d in data],marker='o')
|
||||
axes[0].set(xscale='log',xlabel='Concurrency',ylabel='Successful POST/s',title='Saturation at fixed resources')
|
||||
ax2=axes[0].twinx()
|
||||
ax2.plot([1,4,16,64],[d['p95_ms_mean'] for d in data],marker='s',color='tab:red')
|
||||
ax2.set_ylabel('p95 ms',color='tab:red')
|
||||
names=['saturation-c16','db-0.5','db-1']
|
||||
if all(n in by_name for n in names):
|
||||
axes[1].bar(['0.25 CPU','0.5 CPU','1 CPU'],[by_name[n]['rps_mean'] for n in names])
|
||||
axes[1].set(xlabel='PostgreSQL CPU quota',ylabel='Successful POST/s',title='Bottleneck intervention (4 API, c=16)')
|
||||
fig.savefig(DIRECTORY/'saturation.png',dpi=180)
|
||||
fig.savefig(DIRECTORY/'saturation.svg')
|
||||
plt.close(fig)
|
||||
if all('cache-'+mode in by_name for mode in ('per_request','shared')):
|
||||
fig,axes=plt.subplots(1,2,figsize=(10,4),layout='constrained')
|
||||
names=['cache-per_request','cache-shared']
|
||||
axes[0].bar(['Per-request client','Shared pool'],[by_name[n]['rps_mean'] for n in names], yerr=[by_name[n]['rps_stdev'] for n in names], capsize=5)
|
||||
axes[0].set_ylabel('Successful POST/s')
|
||||
axes[1].bar(['Per-request client','Shared pool'],[by_name[n]['cache_new_connections_mean'] for n in names])
|
||||
axes[1].set_ylabel('New Valkey connections per full run')
|
||||
fig.suptitle('Connection reuse: 2 API × 0.5 CPU, PostgreSQL 1 CPU, c=16')
|
||||
fig.savefig(DIRECTORY/'cache-improvement.png',dpi=180)
|
||||
fig.savefig(DIRECTORY/'cache-improvement.svg')
|
||||
plt.close(fig)
|
||||
print(json.dumps(rows,indent=2))
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reproducible lab3 track 1. Changes ONLY this Compose project's resources.
|
||||
|
||||
Restores the normal two-replica configuration in finally; does not delete volumes.
|
||||
"""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = str(ROOT / 'scripts/compose.sh')
|
||||
DEST = ROOT / 'evidence/scaling'
|
||||
|
||||
|
||||
def call(*args, env=None):
|
||||
return subprocess.check_output(args, cwd=ROOT, env=env, text=True).strip()
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
return call(COMPOSE, *args, env=env)
|
||||
|
||||
|
||||
def inspect(ids):
|
||||
return json.loads(call('docker', 'inspect', *ids))
|
||||
|
||||
|
||||
def wait_ready(env, replicas, base):
|
||||
deadline = time.monotonic()+75
|
||||
while time.monotonic() < deadline:
|
||||
ids = compose('ps', '-q', 'api', env=env).split()
|
||||
if len(ids) == replicas:
|
||||
state = inspect(ids)
|
||||
if all(c['State'].get('Health', {}).get('Status') == 'healthy' for c in state):
|
||||
expected = {c['Config']['Hostname'] for c in state}
|
||||
try:
|
||||
seen = set()
|
||||
for _ in range(32):
|
||||
with urllib.request.urlopen(base+'/health/ready', timeout=2) as response:
|
||||
seen.add(response.headers.get('X-Instance-Id'))
|
||||
if seen == expected:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(.5)
|
||||
raise RuntimeError('replicas/DNS/LB did not converge')
|
||||
|
||||
|
||||
def dependency_stats(env):
|
||||
sql = "SELECT json_build_object('xact_commit',xact_commit,'xact_rollback',xact_rollback,'blks_hit',blks_hit,'blks_read',blks_read,'tup_inserted',tup_inserted,'numbackends',numbackends) FROM pg_stat_database WHERE datname='messenger'"
|
||||
db = json.loads(compose('exec','-T','db','psql','-U','messenger','-d','messenger','-Atc',sql,env=env))
|
||||
info = compose('exec','-T','valkey','valkey-cli','info','stats',env=env)
|
||||
cache = {k: int(v.strip()) for line in info.splitlines() if ':' in line for k,v in [line.split(':',1)] if k in ('total_connections_received','total_commands_processed','keyspace_hits','keyspace_misses')}
|
||||
return {'db':db,'valkey':cache}
|
||||
|
||||
|
||||
class Stats:
|
||||
def __init__(self, ids):
|
||||
self.rows = []
|
||||
self.proc = subprocess.Popen(['docker','stats','--format','{{json .}}','--no-trunc'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True)
|
||||
self.thread = threading.Thread(target=self.read, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def read(self):
|
||||
for line in self.proc.stdout:
|
||||
try:
|
||||
row = json.loads(line[line.find("{"):])
|
||||
row['at_unix'] = time.time()
|
||||
if row.get('Name', '').startswith('mtusi-lab3-'):
|
||||
self.rows.append(row)
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
def close(self):
|
||||
self.proc.kill()
|
||||
self.proc.wait(timeout=5)
|
||||
self.thread.join(timeout=5)
|
||||
|
||||
|
||||
def run_case(case, index, args):
|
||||
env = dict(os.environ, API_CPUS=str(case['api_cpu']), API_MEMORY=case['api_memory'],
|
||||
API_REPLICAS=str(case['replicas']), DB_CPUS=str(case.get('db_cpu',.25)),
|
||||
LB_CPUS=str(case.get('lb_cpu',.25)), DB_POOL_MAX_SIZE=str(case.get('pool',5)),
|
||||
CACHE_CLIENT_MODE=case.get('cache','shared'),
|
||||
COMPOSE_FILE=str(ROOT/'compose.yaml')+os.pathsep+str(ROOT/'compose.research.yaml'))
|
||||
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api='+str(case['replicas']),'api','db','lb'], cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL)
|
||||
wait_ready(env, case['replicas'], args.base_url)
|
||||
ids = compose('ps','-q','api','db','lb','valkey',env=env).split()
|
||||
configs = [{'name':c['Name'], 'id':c['Id'], 'image_id':c['Image'],
|
||||
'cpu_limit':c['HostConfig']['NanoCpus']/1e9, 'memory_bytes':c['HostConfig']['Memory']}
|
||||
for c in inspect(ids)]
|
||||
before = dependency_stats(env)
|
||||
observer = Stats(ids)
|
||||
try:
|
||||
output = compose('run','--rm','-T','--no-deps','--name','mtusi-lab3-research-load','load',
|
||||
'--concurrency',str(case.get('concurrency',args.concurrency)),
|
||||
'--duration',str(args.duration),'--warmup',str(args.warmup),env=env)
|
||||
benchmark = json.loads(output)
|
||||
finally:
|
||||
observer.close()
|
||||
after = dependency_stats(env)
|
||||
cpu = defaultdict(list)
|
||||
for row in observer.rows:
|
||||
if benchmark['measure_started_unix'] <= row['at_unix'] <= benchmark['finished_unix']:
|
||||
cpu[row['Name']].append(float(row['CPUPerc'].rstrip('%')))
|
||||
report = {'case':case,'repeat':index,'benchmark':benchmark,'containers':configs,
|
||||
'cpu_percent_mean':{k:round(sum(v)/len(v),3) for k,v in cpu.items()},
|
||||
'stats_samples':observer.rows,'dependency_before':before,'dependency_after':after}
|
||||
filename = case['name']+'-'+str(index)+'.json'
|
||||
(DEST/filename).write_text(json.dumps(report, indent=2)+'\n')
|
||||
print(json.dumps({'file':filename,'rps':benchmark['successful_rps'],
|
||||
'p95_ms':benchmark['successful_latency_ms']['p95'],
|
||||
'errors':benchmark['errors'],'cpu':report['cpu_percent_mean']}), flush=True)
|
||||
return filename
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url',default=os.getenv('BASE_URL','http://localhost:8083'))
|
||||
parser.add_argument('--duration',type=float,default=12)
|
||||
parser.add_argument('--warmup',type=float,default=3)
|
||||
parser.add_argument('--repeats',type=int,default=3)
|
||||
parser.add_argument('--concurrency',type=int,default=16)
|
||||
parser.add_argument('--suite',choices=['scaling','saturation','improvement','all'],default='all')
|
||||
args=parser.parse_args()
|
||||
DEST.mkdir(parents=True,exist_ok=True)
|
||||
cases=[]
|
||||
if args.suite in ('scaling','all'):
|
||||
for n in (1,2,4):
|
||||
cases.append({'name':f'fixed-{n}','replicas':n,'api_cpu':1/n,'api_memory':str(1024//n)+'m','family':'fixed'})
|
||||
cases.append({'name':f'growing-{n}','replicas':n,'api_cpu':.5,'api_memory':'256m','family':'growing'})
|
||||
if args.suite in ('saturation','all'):
|
||||
for concurrency in (1,4,16,64):
|
||||
cases.append({'name':f'saturation-c{concurrency}','replicas':4,'api_cpu':.5,'api_memory':'256m','concurrency':concurrency,'family':'saturation'})
|
||||
for db_cpu in (.5,1):
|
||||
cases.append({'name':f'db-{db_cpu}','replicas':4,'api_cpu':.5,'api_memory':'256m','db_cpu':db_cpu,'family':'db-quota'})
|
||||
if args.suite in ('improvement','all'):
|
||||
for cache in ('per_request','shared'):
|
||||
cases.append({'name':'cache-'+cache,'replicas':2,'api_cpu':.5,'api_memory':'256m','db_cpu':1,'cache':cache,'family':'cache'})
|
||||
files=[]
|
||||
try:
|
||||
for repeat in range(1,args.repeats+1):
|
||||
ordered=list(cases)
|
||||
random.Random(20261030+repeat).shuffle(ordered)
|
||||
for case in ordered:
|
||||
files.append(run_case(case,repeat,args))
|
||||
finally:
|
||||
# Restore normal config from .env, not the last experimental overrides.
|
||||
subprocess.run([COMPOSE,'up','-d','--no-deps','--scale','api=2','api','db','lb'],cwd=ROOT,check=True)
|
||||
(DEST/('manifest-'+args.suite+'.json')).write_text(json.dumps({'suite':args.suite,'parameters':vars(args),'files':files,'seed_base':20261030},indent=2)+'\n')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
scripts/compose.sh restart db valkey api lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/rolling_update.py
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Replace each API container using a healthy temporary extra replica (surge=1)."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
COMPOSE = str(ROOT / 'scripts/compose.sh')
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def ids():
|
||||
return command(COMPOSE, 'ps', '-q', 'api').split()
|
||||
|
||||
|
||||
def inspect(cid):
|
||||
result = json.loads(command('docker', 'inspect', cid))[0]
|
||||
return {'id': result['Id'], 'hostname': result['Config']['Hostname'],
|
||||
'image_id': result['Image'], 'image': result['Config']['Image'],
|
||||
'status': result['State']['Status'],
|
||||
'health': result['State'].get('Health', {}).get('Status')}
|
||||
|
||||
|
||||
def main():
|
||||
original = ids()
|
||||
if len(original) < 2:
|
||||
raise RuntimeError('rolling update requires at least two running replicas')
|
||||
base = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
evidence = {'before': [inspect(i) for i in original], 'steps': []}
|
||||
try:
|
||||
for old in original:
|
||||
before = set(ids())
|
||||
subprocess.run([COMPOSE, 'up', '-d', '--no-deps', '--no-recreate',
|
||||
'--scale', 'api=' + str(len(original)+1), 'api'], check=True, cwd=ROOT)
|
||||
added = set(ids()) - before
|
||||
if len(added) != 1:
|
||||
raise RuntimeError('expected exactly one spare container')
|
||||
spare = added.pop()
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
info = inspect(spare)
|
||||
# Healthy container must actually be observed behind the public LB.
|
||||
seen = False
|
||||
if info['health'] == 'healthy':
|
||||
try:
|
||||
for _ in range(2 * (len(original) + 1)):
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=2) as response:
|
||||
if response.headers.get('X-Instance-Id') == info['hostname']:
|
||||
seen = True
|
||||
break
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
if seen:
|
||||
break
|
||||
if time.monotonic() >= deadline:
|
||||
raise RuntimeError('spare did not become healthy and reachable via LB')
|
||||
time.sleep(.3)
|
||||
subprocess.run(['docker', 'stop', '--time', '12', old], check=True, stdout=subprocess.DEVNULL)
|
||||
stopped = inspect(old)
|
||||
subprocess.run(['docker', 'rm', old], check=True, stdout=subprocess.DEVNULL)
|
||||
evidence['steps'].append({'removed': stopped, 'replacement': info})
|
||||
evidence['after'] = [inspect(i) for i in ids()]
|
||||
assert len(evidence['after']) == len(original)
|
||||
assert not set(original) & set(ids())
|
||||
evidence['status'] = 'PASS'
|
||||
finally:
|
||||
dest = ROOT / 'evidence/rolling-containers.json'
|
||||
dest.write_text(json.dumps(evidence, indent=2) + '\n')
|
||||
print('PASS: all original API containers replaced; evidence/rolling-containers.json')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?Required API hostname from X-Instance-Id}"
|
||||
for id in $(scripts/compose.sh ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 3 "$id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf '%s\n' 'Target is not a running API replica' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Append tables and evidence-based conclusions to the research methodology."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
rows=json.loads((ROOT/'evidence/scaling/summary.json').read_text())
|
||||
by_name={r['case']:r for r in rows}
|
||||
report=ROOT/'RESEARCH.md'
|
||||
text=report.read_text().split('\n## Результаты\n')[0]
|
||||
text+='\n## Результаты\n\n'
|
||||
text+='Каждая строка — три повтора. Ошибки во всех 42 запусках отсутствовали. RPS±s — среднее и выборочное стандартное отклонение.\n\n'
|
||||
text+='| Серия | Реплик | RPS ± s | min–max RPS | p50 / p95 / p99, мс | CPU БД, % одного CPU |\n| --- | --- | --- | --- | --- | --- |\n'
|
||||
for family in ('fixed','growing'):
|
||||
for n in (1,2,4):
|
||||
r=by_name[f'{family}-{n}']
|
||||
text+=f"| {family} | {n} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['rps_min']:.1f}–{r['rps_max']:.1f} | {r['p50_ms_mean']:.1f} / {r['p95_ms_mean']:.1f} / {r['p99_ms_mean']:.1f} | {r['db_cpu_percent_mean']:.2f} |\n"
|
||||
text+='\n\n\n'
|
||||
text+='При фиксированном бюджете 1→4 реплики снизили throughput, а p95 вырос. Это опровергает ожидание «реплик больше — всегда быстрее». Разделение бюджета между процессами, независимые пулы/healthchecks, синхронный I/O и планирование дробных CPU-квот могут менять эффективность. Вклад каждого фактора здесь отдельно не измерялся, поэтому конкретная причина всей деградации fixed не доказана. При росте бюджета переход 2→4 также не дал устойчивого выигрыша: БД работала около своей полной квоты.\n\n'
|
||||
text+='### Насыщение и вмешательство в ресурс БД\n\n'
|
||||
text+='| Конфигурация: 4 API × 0.5 CPU | RPS ± s | p95, мс | CPU БД, % одного CPU | CPU LB / клиента, % |\n| --- | --- | --- | --- | --- |\n'
|
||||
for name in ('saturation-c1','saturation-c4','saturation-c16','saturation-c64','db-0.5','db-1'):
|
||||
r=by_name[name]
|
||||
text+=f"| {name} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['p95_ms_mean']:.1f} | {r['db_cpu_percent_mean']:.2f} | {r['lb_cpu_percent_mean']:.2f} / {r['client_cpu_percent_mean']:.2f} |\n"
|
||||
a,b=by_name['saturation-c16'],by_name['db-1']
|
||||
text+=f"\nПри c=4 БД уже близка к квоте 25%; максимум среднего throughput среди исследованных c находится при c=16 ({a['rps_mean']:.1f} POST/с). Рост c до 64 не дал прироста, p95 вырос до {by_name['saturation-c64']['p95_ms_mean']:.1f} мс. Это эмпирическая область насыщения около 400–500 POST/с для данной конфигурации, не точная универсальная граница. При увеличении квоты только БД до 1 CPU средний throughput вырос в {b['rps_mean']/a['rps_mean']:.2f} раза, до {b['rps_mean']:.1f} POST/с. Это подтверждает ограничение общей зависимости в исходном стенде.\n\n"
|
||||
text+='LB и генератор существенно ниже своих CPU-лимитов 25% и 100% соответственно; их CPU не объясняет исходное плато. При DB=1 CPU сама БД уже не достигает квоты; ограничение сдвигается. Точное новое узкое место (Python/I/O/планировщик/пул) этим экспериментом не локализовано. Один процесс API выполняет синхронные операции БД в async-обработчике, поэтому добавление размера пула не гарантирует параллелизм. Не заявляется, что достигнута предельная мощность PostgreSQL или устранены все bottleneck.\n\n'
|
||||
text+='\n\n'
|
||||
text+='### Подтверждённое улучшение: переиспользование соединений Valkey\n\n'
|
||||
text+='Объект Redis теперь создаётся один раз на Store/процесс и использует библиотечный connection pool; закрывается при shutdown. Для контрольного эксперимента CACHE_CLIENT_MODE=per_request воспроизводит прежнее создание клиента на каждую операцию. По умолчанию выбран shared. Проверка идентичности/expiry/revocation в PostgreSQL не меняется.\n\n'
|
||||
text+='| Вариант | RPS ± s | min–max RPS | p95, мс | Новых соединений Valkey за весь запуск, среднее |\n| --- | --- | --- | --- | --- |\n'
|
||||
for name in ('cache-per_request','cache-shared'):
|
||||
r=by_name[name]
|
||||
text+=f"| {name} | {r['rps_mean']:.1f} ± {r['rps_stdev']:.1f} | {r['rps_min']:.1f}–{r['rps_max']:.1f} | {r['p95_ms_mean']:.1f} | {r['cache_new_connections_mean']:.3f} |\n"
|
||||
a,b=by_name['cache-per_request'],by_name['cache-shared']
|
||||
text+=f"\nВ этих трёх повторах средний throughput вырос на {100*(b['rps_mean']/a['rps_mean']-1):.1f}%, средний p95 снизился на {100*(1-b['p95_ms_mean']/a['p95_ms_mean']):.1f}%. Число новых соединений снизилось на {100*(1-b['cache_new_connections_mean']/a['cache_new_connections_mean']):.3f}%. Интервалы min–max throughput перекрываются, статистическая значимость не проверялась; гарантировать прирост 31% для иной нагрузки нельзя. Устойчиво подтверждено устранение почти всех повторных подключений.\n\n"
|
||||
text+='\n\n'
|
||||
text+='## Ограничения и практический вывод\n\n'
|
||||
text+='Три коротких повтора по десять секунд, короткий прогрев, растущие данные и VM дают заметный разброс. Не контролировались page cache, фазы CFS-квот, частота CPU и фоновая активность ОС. Стандартное отклонение не является доверительным интервалом. Измеряется одна короткая POST-нагрузка с действующими сессиями, а не смесь логинов, чтения больших историй и файлов. Нет open-loop arrival rate, длинного soak-теста, статистического теста значимости или доказательства исчерпания RAM. Raw summaries содержат memory usage/limits, а не профили heap.\n\n'
|
||||
text+='До увеличения числа API нужно измерять общую зависимость: в этом стенде DB CPU объясняет плато growing, а дробление фиксированного CPU не помогает. Подтверждены два практических эффекта: снятие искусственно тесной квоты БД увеличивает пропускную способность; общий пул Valkey устраняет churn соединений и улучшает средние показатели этого профиля. При этом в обычной конфигурации сохранены две API для отказоустойчивости процесса, а не выбрана единственная реплика ради лучшего числа в fixed-серии.\n'
|
||||
report.write_text(text)
|
||||
Reference in New Issue
Block a user