- X-Instance-Id, обязательный Idempotency-Key (PostgreSQL, одна транзакция) - HAProxy + 2..N реплик API, readiness/drain по SIGTERM, однократные миграции - скрипты stop-one/restore/restart/rolling-update/collect-evidence - трек 1: параметризованный стенд, cmd/bench, оркестратор и анализ - workflow реального прогона Docker-стенда на GitHub Actions Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
317 lines
15 KiB
Python
317 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""Оркестратор экспериментов трека 1 (цена масштабирования).
|
||
|
||
Для каждой серии/конфигурации из experiments/track1.json:
|
||
1. поднимает отдельный compose-проект (по умолчанию mtusi2026lab3-bench, порт 18080)
|
||
с переменными конфигурации (число реплик, CPU/RAM, пул, LB, режим кэша);
|
||
2. ждёт, пока через LB видны все N реплик;
|
||
3. для каждого уровня нагрузки и повтора запускает bin/bench (Go), параллельно
|
||
раз в ~1 s снимает docker stats всех контейнеров и pg_stat_activity, до и
|
||
после — счётчики пула pgx каждой реплики (внутренний :9090, через docker exec);
|
||
4. дописывает одну JSON-строку на прогон в evidence/track1/raw.jsonl.
|
||
|
||
Анализ и графики: scripts/track1_report.py. Только stdlib + docker CLI + go.
|
||
|
||
Примеры:
|
||
python scripts/track1.py --quick # быстрый прогон всех серий
|
||
python scripts/track1.py --series A_fixed_total # одна серия, полная методика
|
||
python scripts/track1.py --resume # продолжить после обрыва
|
||
python scripts/track1.py --cleanup # в конце удалить bench-проект с volumes
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import statistics
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
BENCH = ROOT / 'bin' / ('bench.exe' if os.name == 'nt' else 'bench')
|
||
|
||
|
||
def sh(args, env=None, check=True, capture=True, timeout=600):
|
||
r = subprocess.run(args, cwd=ROOT, env=env, capture_output=capture, text=True, timeout=timeout)
|
||
if check and r.returncode != 0:
|
||
raise SystemExit('command failed: %s\n%s' % (' '.join(args[:6]), (r.stderr or '')[-2000:]))
|
||
return (r.stdout or '').strip()
|
||
|
||
|
||
class Stack:
|
||
def __init__(self, project, port):
|
||
self.project, self.port = project, port
|
||
self.base = 'http://127.0.0.1:%d' % port
|
||
|
||
def compose(self, *args, env=None, **kw):
|
||
return sh(['docker', 'compose', '-p', self.project, *args], env=env, **kw)
|
||
|
||
def up(self, env, build):
|
||
args = ['up', '-d', '--wait', '--remove-orphans']
|
||
if build:
|
||
args.append('--build')
|
||
n = env['API_REPLICAS']
|
||
self.compose(*args, '--scale', 'api=' + n, env=env, timeout=900)
|
||
|
||
def containers(self):
|
||
out = {}
|
||
for cid in self.compose('ps', '-q').split():
|
||
svc = sh(['docker', 'inspect', '-f', '{{index .Config.Labels "com.docker.compose.service"}}', cid])
|
||
out[cid[:12]] = svc
|
||
return out
|
||
|
||
def instances_via_lb(self, tries):
|
||
seen = set()
|
||
for _ in range(tries):
|
||
try:
|
||
with urllib.request.urlopen(self.base + '/health/ready', timeout=2) as r:
|
||
seen.add(r.headers.get('X-Instance-Id'))
|
||
except OSError:
|
||
pass
|
||
return seen
|
||
|
||
def wait_balanced(self, n, timeout=90):
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
if len(self.instances_via_lb(4 * n + 4)) >= n:
|
||
return
|
||
time.sleep(1)
|
||
raise SystemExit('LB does not route to %s replicas' % n)
|
||
|
||
def pool_stats(self, containers):
|
||
res = {}
|
||
for cid, svc in containers.items():
|
||
if svc != 'api':
|
||
continue
|
||
out = sh(['docker', 'exec', cid, 'wget', '-qO-', 'http://127.0.0.1:9090/internal/stats'], check=False)
|
||
try:
|
||
res[cid] = json.loads(out)['pool']
|
||
except (ValueError, KeyError):
|
||
pass
|
||
return res
|
||
|
||
def pad_redis(self, keys):
|
||
if keys <= 0:
|
||
return
|
||
have = int(self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL',
|
||
"return #redis.call('KEYS','pad:*')", '0') or 0)
|
||
if have >= keys:
|
||
return
|
||
# Фоновый keyspace (имитация активных сессий/прочих ключей). Только синтетика.
|
||
self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL',
|
||
"for i=1,tonumber(ARGV[1]) do redis.call('SET','pad:'..i,'x','EX',604800) end return 1",
|
||
'0', str(keys))
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
"""docker stats + pg_stat_activity примерно раз в секунду."""
|
||
|
||
def __init__(self, stack, containers):
|
||
super().__init__(daemon=True)
|
||
self.stack, self.containers = stack, containers
|
||
self.samples, self.pg = [], []
|
||
self.stop = threading.Event()
|
||
|
||
def run(self):
|
||
ids = list(self.containers)
|
||
while not self.stop.is_set():
|
||
t = time.monotonic()
|
||
out = sh(['docker', 'stats', '--no-stream', '--format', '{{.ID}} {{.CPUPerc}} {{.MemUsage}}', *ids], check=False)
|
||
for line in out.splitlines():
|
||
parts = line.split()
|
||
if len(parts) < 2:
|
||
continue
|
||
try:
|
||
cpu = float(parts[1].rstrip('%'))
|
||
except ValueError:
|
||
continue
|
||
mem = parts[2] if len(parts) > 2 else ''
|
||
self.samples.append((t, parts[0][:12], cpu, mem))
|
||
q = ("select count(*) filter (where state='active'), count(*) "
|
||
"from pg_stat_activity where datname='messenger' and pid<>pg_backend_pid()")
|
||
pg = self.stack.compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||
'-tAc', q, check=False)
|
||
try:
|
||
a, total = pg.split('|')
|
||
self.pg.append((t, int(a), int(total)))
|
||
except ValueError:
|
||
pass
|
||
|
||
|
||
def mem_mib(s):
|
||
s = s.strip()
|
||
for unit, k in (('GiB', 1024), ('MiB', 1), ('KiB', 1 / 1024), ('B', 1 / 1024 / 1024)):
|
||
if s.endswith(unit):
|
||
try:
|
||
return float(s[:-len(unit)]) * k
|
||
except ValueError:
|
||
return None
|
||
return None
|
||
|
||
|
||
def summarize_resources(sampler, containers, env, t_from, t_to):
|
||
limits = {'api': float(env['API_CPUS']), 'postgres': float(env['PG_CPUS']),
|
||
'lb': float(env['LB_CPUS']), 'redis': float(env['REDIS_CPUS'])}
|
||
per = {}
|
||
for t, cid, cpu, mem in sampler.samples:
|
||
if t_from <= t <= t_to and cid in containers:
|
||
per.setdefault(cid, []).append((cpu, mem_mib(mem)))
|
||
roles = {}
|
||
for cid, vals in per.items():
|
||
svc = containers[cid]
|
||
cpus = [v[0] for v in vals]
|
||
mems = [v[1] for v in vals if v[1] is not None]
|
||
roles.setdefault(svc, []).append({'cpu_mean': statistics.mean(cpus), 'cpu_max': max(cpus),
|
||
'mem_max_mib': max(mems) if mems else None})
|
||
out = {}
|
||
for svc, items in roles.items():
|
||
lim = limits.get(svc)
|
||
mean = statistics.mean(i['cpu_mean'] for i in items)
|
||
out[svc] = {
|
||
'containers': len(items),
|
||
'cpu_pct_mean_per_container': round(mean, 1),
|
||
'cpu_pct_max_per_container': round(max(i['cpu_max'] for i in items), 1),
|
||
# 100% = весь лимит контейнера; ≥85% считаем насыщением.
|
||
'cpu_utilization_of_limit': round(mean / (lim * 100), 3) if lim else None,
|
||
'mem_max_mib': round(max((i['mem_max_mib'] or 0) for i in items), 1),
|
||
}
|
||
pg = [x for x in sampler.pg if t_from <= x[0] <= t_to]
|
||
if pg:
|
||
out['postgres_connections'] = {'active_max': max(x[1] for x in pg),
|
||
'active_mean': round(statistics.mean(x[1] for x in pg), 1),
|
||
'total_max': max(x[2] for x in pg)}
|
||
return out
|
||
|
||
|
||
def pool_delta(before, after):
|
||
acq = empty = dur = 0
|
||
max_conns = 0
|
||
for cid, a in after.items():
|
||
b = before.get(cid, {})
|
||
acq += a['acquire_count'] - b.get('acquire_count', 0)
|
||
empty += a['empty_acquire_count'] - b.get('empty_acquire_count', 0)
|
||
dur += a['acquire_duration_ms'] - b.get('acquire_duration_ms', 0)
|
||
max_conns += a['max_conns']
|
||
return {'acquires': acq, 'empty_acquire_ratio': round(empty / acq, 4) if acq else None,
|
||
'avg_acquire_wait_ms': round(dur / acq, 3) if acq else None, 'pool_max_conns_total': max_conns}
|
||
|
||
|
||
def done_keys(path):
|
||
keys = set()
|
||
if path.exists():
|
||
for line in path.read_text(encoding='utf-8').splitlines():
|
||
try:
|
||
r = json.loads(line)
|
||
keys.add((r['series'], r['config'], r['concurrency'], r['repeat']))
|
||
except (ValueError, KeyError):
|
||
pass
|
||
return keys
|
||
|
||
|
||
def main():
|
||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||
p.add_argument('--experiments', default='experiments/track1.json')
|
||
p.add_argument('--series', default='', help='comma-separated series names; default all')
|
||
p.add_argument('--output', default='evidence/track1/raw.jsonl')
|
||
p.add_argument('--quick', action='store_true', help='levels 4,16,64; 15s; 1 repeat')
|
||
p.add_argument('--resume', action='store_true')
|
||
p.add_argument('--configs', default='', help='comma-separated config labels to run; default all')
|
||
p.add_argument('--levels', default='', help='override levels, e.g. 4,16,64')
|
||
p.add_argument('--duration', default='', help='override measured duration, e.g. 15s')
|
||
p.add_argument('--warmup', default='', help='override warmup, e.g. 3s')
|
||
p.add_argument('--repeats', type=int, default=0, help='override repeats')
|
||
p.add_argument('--cleanup', action='store_true', help='docker compose down -v for the bench project at the end')
|
||
args = p.parse_args()
|
||
|
||
exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8'))
|
||
out = ROOT / args.output
|
||
out.parent.mkdir(parents=True, exist_ok=True)
|
||
skip = done_keys(out) if args.resume else set()
|
||
wanted = {s for s in args.series.split(',') if s}
|
||
stack = Stack(exp['project'], exp['lb_port'])
|
||
|
||
probe = subprocess.run(['docker', 'version', '--format', '{{.Server.Version}}'], capture_output=True, text=True)
|
||
if probe.returncode != 0:
|
||
raise SystemExit('Docker Engine недоступен (docker version: %s). '
|
||
'Запустите Docker Desktop и дождитесь "Engine running"; '
|
||
'на Windows нужны включённая виртуализация и WSL2.' % (probe.stderr.strip().splitlines() or ['?'])[-1])
|
||
sh(['go', 'build', '-o', str(BENCH), './cmd/bench'])
|
||
host = {'host_cpus': os.cpu_count(), 'docker': sh(['docker', 'version', '--format', '{{.Server.Version}}'], check=False),
|
||
'docker_cpus': sh(['docker', 'info', '--format', '{{.NCPU}}'], check=False),
|
||
'docker_mem_bytes': sh(['docker', 'info', '--format', '{{.MemTotal}}'], check=False)}
|
||
print('host:', json.dumps(host))
|
||
built = False
|
||
|
||
for series in exp['series']:
|
||
if wanted and series['name'] not in wanted:
|
||
continue
|
||
load = dict(exp['load'])
|
||
load['levels'] = series.get('levels', load['levels'])
|
||
load['repeats'] = series.get('repeats', load['repeats'])
|
||
if args.quick:
|
||
load.update(levels=[4, 16, 64] if series['name'] != 'D_improvement' else [32], duration='15s',
|
||
warmup='3s', repeats=1)
|
||
if args.levels:
|
||
load['levels'] = [int(x) for x in args.levels.split(',')]
|
||
for k in ('duration', 'warmup', 'repeats'):
|
||
if getattr(args, k):
|
||
load[k] = getattr(args, k)
|
||
only = {c for c in args.configs.split(',') if c}
|
||
for cfg in series['configs']:
|
||
if only and cfg['label'] not in only:
|
||
continue
|
||
env = dict(os.environ)
|
||
env.update(exp['base_env'])
|
||
env.update({k: v for k, v in cfg.items() if k != 'label'})
|
||
env.setdefault('API_MEMORY', '256M')
|
||
env['LB_PORT'] = str(exp['lb_port'])
|
||
env['LB_STATS_PORT'] = str(exp['lb_port'] + 324)
|
||
todo = [(lvl, rep) for lvl in load['levels'] for rep in range(load['repeats'])
|
||
if (series['name'], cfg['label'], lvl, rep) not in skip]
|
||
if not todo:
|
||
continue
|
||
print('== %s / %s' % (series['name'], cfg['label']), flush=True)
|
||
stack.up(env, build=not built)
|
||
built = True
|
||
n = int(env['API_REPLICAS'])
|
||
stack.wait_balanced(n)
|
||
stack.pad_redis(exp.get('redis_padding_keys', 0))
|
||
containers = stack.containers()
|
||
config_env = {k: env[k] for k in ('API_REPLICAS', 'API_CPUS', 'API_MEMORY', 'PG_CPUS', 'LB_CPUS',
|
||
'REDIS_CPUS', 'DB_POOL_MAX', 'CACHE_INVALIDATION', 'LB_BALANCE')}
|
||
for lvl, rep in todo:
|
||
before = stack.pool_stats(containers)
|
||
sampler = Sampler(stack, containers)
|
||
sampler.start()
|
||
t_start = time.monotonic()
|
||
warm = float(load['warmup'].rstrip('s'))
|
||
dur = float(load['duration'].rstrip('s'))
|
||
line = sh([str(BENCH), '-base-url', stack.base, '-concurrency', str(lvl),
|
||
'-duration', load['duration'], '-warmup', load['warmup'],
|
||
'-write-ratio', str(load['write_ratio']), '-chats', str(load['chats']),
|
||
'-label', '%s/%s' % (series['name'], cfg['label'])], timeout=dur + warm + 300)
|
||
sampler.stop.set()
|
||
sampler.join(timeout=30)
|
||
after = stack.pool_stats(containers)
|
||
bench = json.loads(line.splitlines()[-1])
|
||
rec = {'series': series['name'], 'config': cfg['label'], 'concurrency': lvl, 'repeat': rep,
|
||
'load': {k: load[k] for k in ('duration', 'warmup', 'write_ratio', 'chats')},
|
||
'env': config_env, 'host': host, 'bench': bench,
|
||
'resources': summarize_resources(sampler, containers, env, t_start + warm, t_start + warm + dur),
|
||
'pool': pool_delta(before, after)}
|
||
with out.open('a', encoding='utf-8') as f:
|
||
f.write(json.dumps(rec, ensure_ascii=False) + '\n')
|
||
print(' c=%-3d rep=%d ok_rps=%.1f p95=%.1fms err=%.4f' % (
|
||
lvl, rep, bench['ok_rps'], bench['latency_ms']['p95'], bench['error_ratio']), flush=True)
|
||
|
||
if args.cleanup:
|
||
# Удаляет ТОЛЬКО bench-проект и его синтетические volumes.
|
||
stack.compose('down', '-v')
|
||
print('raw results:', out)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
sys.exit(main())
|