#!/usr/bin/env python3 """Оркестратор экспериментов трека 1 (цена масштабирования). Для каждой серии/конфигурации из experiments/track1.json: 1. поднимает отдельный compose-проект (по умолчанию mtusi2026lab3-bench, порт 18080) с переменными конфигурации (число реплик, CPU/RAM, пул, LB, режим кэша); 2. ждёт, пока через LB видны все N реплик; 3. для каждого уровня нагрузки и повтора запускает bin/bench (Go), параллельно раз в ~1 s снимает docker stats всех контейнеров и pg_stat_activity, до и после — счётчики пула pgx каждой реплики (внутренний :9090, через docker exec); 4. дописывает одну JSON-строку на прогон в evidence/track1/raw.jsonl. Анализ и графики: scripts/track1_report.py. Только stdlib + docker CLI + go. Примеры: python scripts/track1.py --quick # быстрый прогон всех серий python scripts/track1.py --series A_fixed_total # одна серия, полная методика python scripts/track1.py --resume # продолжить после обрыва python scripts/track1.py --cleanup # в конце удалить bench-проект с volumes """ import argparse import json import os from pathlib import Path import statistics import subprocess import sys import threading import time import urllib.request ROOT = Path(__file__).resolve().parents[1] BENCH = ROOT / 'bin' / ('bench.exe' if os.name == 'nt' else 'bench') def sh(args, env=None, check=True, capture=True, timeout=600): r = subprocess.run(args, cwd=ROOT, env=env, capture_output=capture, text=True, timeout=timeout) if check and r.returncode != 0: raise SystemExit('command failed: %s\n%s' % (' '.join(args[:6]), (r.stderr or '')[-2000:])) return (r.stdout or '').strip() class Stack: def __init__(self, project, port): self.project, self.port = project, port self.base = 'http://127.0.0.1:%d' % port def compose(self, *args, env=None, **kw): return sh(['docker', 'compose', '-p', self.project, *args], env=env, **kw) def up(self, env, build): args = ['up', '-d', '--wait', '--remove-orphans'] if build: args.append('--build') n = env['API_REPLICAS'] self.compose(*args, '--scale', 'api=' + n, env=env, timeout=900) def containers(self): out = {} for cid in self.compose('ps', '-q').split(): svc = sh(['docker', 'inspect', '-f', '{{index .Config.Labels "com.docker.compose.service"}}', cid]) out[cid[:12]] = svc return out def instances_via_lb(self, tries): seen = set() for _ in range(tries): try: with urllib.request.urlopen(self.base + '/health/ready', timeout=2) as r: seen.add(r.headers.get('X-Instance-Id')) except OSError: pass return seen def wait_balanced(self, n, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if len(self.instances_via_lb(4 * n + 4)) >= n: return time.sleep(1) raise SystemExit('LB does not route to %s replicas' % n) def pool_stats(self, containers): res = {} for cid, svc in containers.items(): if svc != 'api': continue out = sh(['docker', 'exec', cid, 'wget', '-qO-', 'http://127.0.0.1:9090/internal/stats'], check=False) try: res[cid] = json.loads(out)['pool'] except (ValueError, KeyError): pass return res def pad_redis(self, keys): if keys <= 0: return have = int(self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', "return #redis.call('KEYS','pad:*')", '0') or 0) if have >= keys: return # Фоновый keyspace (имитация активных сессий/прочих ключей). Только синтетика. self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', "for i=1,tonumber(ARGV[1]) do redis.call('SET','pad:'..i,'x','EX',604800) end return 1", '0', str(keys)) class Sampler(threading.Thread): """docker stats + pg_stat_activity примерно раз в секунду.""" def __init__(self, stack, containers): super().__init__(daemon=True) self.stack, self.containers = stack, containers self.samples, self.pg = [], [] self.stop = threading.Event() def run(self): ids = list(self.containers) while not self.stop.is_set(): t = time.monotonic() out = sh(['docker', 'stats', '--no-stream', '--format', '{{.ID}} {{.CPUPerc}} {{.MemUsage}}', *ids], check=False) for line in out.splitlines(): parts = line.split() if len(parts) < 2: continue try: cpu = float(parts[1].rstrip('%')) except ValueError: continue mem = parts[2] if len(parts) > 2 else '' self.samples.append((t, parts[0][:12], cpu, mem)) q = ("select count(*) filter (where state='active'), count(*) " "from pg_stat_activity where datname='messenger' and pid<>pg_backend_pid()") pg = self.stack.compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', q, check=False) try: a, total = pg.split('|') self.pg.append((t, int(a), int(total))) except ValueError: pass def mem_mib(s): s = s.strip() for unit, k in (('GiB', 1024), ('MiB', 1), ('KiB', 1 / 1024), ('B', 1 / 1024 / 1024)): if s.endswith(unit): try: return float(s[:-len(unit)]) * k except ValueError: return None return None def summarize_resources(sampler, containers, env, t_from, t_to): limits = {'api': float(env['API_CPUS']), 'postgres': float(env['PG_CPUS']), 'lb': float(env['LB_CPUS']), 'redis': float(env['REDIS_CPUS'])} per = {} for t, cid, cpu, mem in sampler.samples: if t_from <= t <= t_to and cid in containers: per.setdefault(cid, []).append((cpu, mem_mib(mem))) roles = {} for cid, vals in per.items(): svc = containers[cid] cpus = [v[0] for v in vals] mems = [v[1] for v in vals if v[1] is not None] roles.setdefault(svc, []).append({'cpu_mean': statistics.mean(cpus), 'cpu_max': max(cpus), 'mem_max_mib': max(mems) if mems else None}) out = {} for svc, items in roles.items(): lim = limits.get(svc) mean = statistics.mean(i['cpu_mean'] for i in items) out[svc] = { 'containers': len(items), 'cpu_pct_mean_per_container': round(mean, 1), 'cpu_pct_max_per_container': round(max(i['cpu_max'] for i in items), 1), # 100% = весь лимит контейнера; ≥85% считаем насыщением. 'cpu_utilization_of_limit': round(mean / (lim * 100), 3) if lim else None, 'mem_max_mib': round(max((i['mem_max_mib'] or 0) for i in items), 1), } pg = [x for x in sampler.pg if t_from <= x[0] <= t_to] if pg: out['postgres_connections'] = {'active_max': max(x[1] for x in pg), 'active_mean': round(statistics.mean(x[1] for x in pg), 1), 'total_max': max(x[2] for x in pg)} return out def pool_delta(before, after): acq = empty = dur = 0 max_conns = 0 for cid, a in after.items(): b = before.get(cid, {}) acq += a['acquire_count'] - b.get('acquire_count', 0) empty += a['empty_acquire_count'] - b.get('empty_acquire_count', 0) dur += a['acquire_duration_ms'] - b.get('acquire_duration_ms', 0) max_conns += a['max_conns'] return {'acquires': acq, 'empty_acquire_ratio': round(empty / acq, 4) if acq else None, 'avg_acquire_wait_ms': round(dur / acq, 3) if acq else None, 'pool_max_conns_total': max_conns} def done_keys(path): keys = set() if path.exists(): for line in path.read_text(encoding='utf-8').splitlines(): try: r = json.loads(line) keys.add((r['series'], r['config'], r['concurrency'], r['repeat'])) except (ValueError, KeyError): pass return keys def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('--experiments', default='experiments/track1.json') p.add_argument('--series', default='', help='comma-separated series names; default all') p.add_argument('--output', default='evidence/track1/raw.jsonl') p.add_argument('--quick', action='store_true', help='levels 4,16,64; 15s; 1 repeat') p.add_argument('--resume', action='store_true') p.add_argument('--configs', default='', help='comma-separated config labels to run; default all') p.add_argument('--levels', default='', help='override levels, e.g. 4,16,64') p.add_argument('--duration', default='', help='override measured duration, e.g. 15s') p.add_argument('--warmup', default='', help='override warmup, e.g. 3s') p.add_argument('--repeats', type=int, default=0, help='override repeats') p.add_argument('--cleanup', action='store_true', help='docker compose down -v for the bench project at the end') args = p.parse_args() exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8')) out = ROOT / args.output out.parent.mkdir(parents=True, exist_ok=True) skip = done_keys(out) if args.resume else set() wanted = {s for s in args.series.split(',') if s} stack = Stack(exp['project'], exp['lb_port']) probe = subprocess.run(['docker', 'version', '--format', '{{.Server.Version}}'], capture_output=True, text=True) if probe.returncode != 0: raise SystemExit('Docker Engine недоступен (docker version: %s). ' 'Запустите Docker Desktop и дождитесь "Engine running"; ' 'на Windows нужны включённая виртуализация и WSL2.' % (probe.stderr.strip().splitlines() or ['?'])[-1]) sh(['go', 'build', '-o', str(BENCH), './cmd/bench']) host = {'host_cpus': os.cpu_count(), 'docker': sh(['docker', 'version', '--format', '{{.Server.Version}}'], check=False), 'docker_cpus': sh(['docker', 'info', '--format', '{{.NCPU}}'], check=False), 'docker_mem_bytes': sh(['docker', 'info', '--format', '{{.MemTotal}}'], check=False)} print('host:', json.dumps(host)) built = False for series in exp['series']: if wanted and series['name'] not in wanted: continue load = dict(exp['load']) load['levels'] = series.get('levels', load['levels']) load['repeats'] = series.get('repeats', load['repeats']) if args.quick: load.update(levels=[4, 16, 64] if series['name'] != 'D_improvement' else [32], duration='15s', warmup='3s', repeats=1) if args.levels: load['levels'] = [int(x) for x in args.levels.split(',')] for k in ('duration', 'warmup', 'repeats'): if getattr(args, k): load[k] = getattr(args, k) only = {c for c in args.configs.split(',') if c} for cfg in series['configs']: if only and cfg['label'] not in only: continue env = dict(os.environ) env.update(exp['base_env']) env.update({k: v for k, v in cfg.items() if k != 'label'}) env.setdefault('API_MEMORY', '256M') env['LB_PORT'] = str(exp['lb_port']) env['LB_STATS_PORT'] = str(exp['lb_port'] + 324) todo = [(lvl, rep) for lvl in load['levels'] for rep in range(load['repeats']) if (series['name'], cfg['label'], lvl, rep) not in skip] if not todo: continue print('== %s / %s' % (series['name'], cfg['label']), flush=True) stack.up(env, build=not built) built = True n = int(env['API_REPLICAS']) stack.wait_balanced(n) stack.pad_redis(exp.get('redis_padding_keys', 0)) containers = stack.containers() config_env = {k: env[k] for k in ('API_REPLICAS', 'API_CPUS', 'API_MEMORY', 'PG_CPUS', 'LB_CPUS', 'REDIS_CPUS', 'DB_POOL_MAX', 'CACHE_INVALIDATION', 'LB_BALANCE')} for lvl, rep in todo: before = stack.pool_stats(containers) sampler = Sampler(stack, containers) sampler.start() t_start = time.monotonic() warm = float(load['warmup'].rstrip('s')) dur = float(load['duration'].rstrip('s')) line = sh([str(BENCH), '-base-url', stack.base, '-concurrency', str(lvl), '-duration', load['duration'], '-warmup', load['warmup'], '-write-ratio', str(load['write_ratio']), '-chats', str(load['chats']), '-label', '%s/%s' % (series['name'], cfg['label'])], timeout=dur + warm + 300) sampler.stop.set() sampler.join(timeout=30) after = stack.pool_stats(containers) bench = json.loads(line.splitlines()[-1]) rec = {'series': series['name'], 'config': cfg['label'], 'concurrency': lvl, 'repeat': rep, 'load': {k: load[k] for k in ('duration', 'warmup', 'write_ratio', 'chats')}, 'env': config_env, 'host': host, 'bench': bench, 'resources': summarize_resources(sampler, containers, env, t_start + warm, t_start + warm + dur), 'pool': pool_delta(before, after)} with out.open('a', encoding='utf-8') as f: f.write(json.dumps(rec, ensure_ascii=False) + '\n') print(' c=%-3d rep=%d ok_rps=%.1f p95=%.1fms err=%.4f' % ( lvl, rep, bench['ok_rps'], bench['latency_ms']['p95'], bench['error_ratio']), flush=True) if args.cleanup: # Удаляет ТОЛЬКО bench-проект и его синтетические volumes. stack.compose('down', '-v') print('raw results:', out) if __name__ == '__main__': sys.exit(main())