Files
mtusi2026lab3/scripts/track1.py
T
Weinmaier_AndrewandClaude Opus 5.5 2e7a80be92 lab3: балансировка, идемпотентность, отказ реплики, трек 1
- X-Instance-Id, обязательный Idempotency-Key (PostgreSQL, одна транзакция)
- HAProxy + 2..N реплик API, readiness/drain по SIGTERM, однократные миграции
- скрипты stop-one/restore/restart/rolling-update/collect-evidence
- трек 1: параметризованный стенд, cmd/bench, оркестратор и анализ
- workflow реального прогона Docker-стенда на GitHub Actions

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 17:24:54 +03:00

317 lines
15 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Оркестратор экспериментов трека 1 (цена масштабирования).
Для каждой серии/конфигурации из experiments/track1.json:
1. поднимает отдельный compose-проект (по умолчанию mtusi2026lab3-bench, порт 18080)
с переменными конфигурации (число реплик, CPU/RAM, пул, LB, режим кэша);
2. ждёт, пока через LB видны все N реплик;
3. для каждого уровня нагрузки и повтора запускает bin/bench (Go), параллельно
раз в ~1 s снимает docker stats всех контейнеров и pg_stat_activity, до и
после — счётчики пула pgx каждой реплики (внутренний :9090, через docker exec);
4. дописывает одну JSON-строку на прогон в evidence/track1/raw.jsonl.
Анализ и графики: scripts/track1_report.py. Только stdlib + docker CLI + go.
Примеры:
python scripts/track1.py --quick # быстрый прогон всех серий
python scripts/track1.py --series A_fixed_total # одна серия, полная методика
python scripts/track1.py --resume # продолжить после обрыва
python scripts/track1.py --cleanup # в конце удалить bench-проект с volumes
"""
import argparse
import json
import os
from pathlib import Path
import statistics
import subprocess
import sys
import threading
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
BENCH = ROOT / 'bin' / ('bench.exe' if os.name == 'nt' else 'bench')
def sh(args, env=None, check=True, capture=True, timeout=600):
r = subprocess.run(args, cwd=ROOT, env=env, capture_output=capture, text=True, timeout=timeout)
if check and r.returncode != 0:
raise SystemExit('command failed: %s\n%s' % (' '.join(args[:6]), (r.stderr or '')[-2000:]))
return (r.stdout or '').strip()
class Stack:
def __init__(self, project, port):
self.project, self.port = project, port
self.base = 'http://127.0.0.1:%d' % port
def compose(self, *args, env=None, **kw):
return sh(['docker', 'compose', '-p', self.project, *args], env=env, **kw)
def up(self, env, build):
args = ['up', '-d', '--wait', '--remove-orphans']
if build:
args.append('--build')
n = env['API_REPLICAS']
self.compose(*args, '--scale', 'api=' + n, env=env, timeout=900)
def containers(self):
out = {}
for cid in self.compose('ps', '-q').split():
svc = sh(['docker', 'inspect', '-f', '{{index .Config.Labels "com.docker.compose.service"}}', cid])
out[cid[:12]] = svc
return out
def instances_via_lb(self, tries):
seen = set()
for _ in range(tries):
try:
with urllib.request.urlopen(self.base + '/health/ready', timeout=2) as r:
seen.add(r.headers.get('X-Instance-Id'))
except OSError:
pass
return seen
def wait_balanced(self, n, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if len(self.instances_via_lb(4 * n + 4)) >= n:
return
time.sleep(1)
raise SystemExit('LB does not route to %s replicas' % n)
def pool_stats(self, containers):
res = {}
for cid, svc in containers.items():
if svc != 'api':
continue
out = sh(['docker', 'exec', cid, 'wget', '-qO-', 'http://127.0.0.1:9090/internal/stats'], check=False)
try:
res[cid] = json.loads(out)['pool']
except (ValueError, KeyError):
pass
return res
def pad_redis(self, keys):
if keys <= 0:
return
have = int(self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL',
"return #redis.call('KEYS','pad:*')", '0') or 0)
if have >= keys:
return
# Фоновый keyspace (имитация активных сессий/прочих ключей). Только синтетика.
self.compose('exec', '-T', 'redis', 'redis-cli', 'EVAL',
"for i=1,tonumber(ARGV[1]) do redis.call('SET','pad:'..i,'x','EX',604800) end return 1",
'0', str(keys))
class Sampler(threading.Thread):
"""docker stats + pg_stat_activity примерно раз в секунду."""
def __init__(self, stack, containers):
super().__init__(daemon=True)
self.stack, self.containers = stack, containers
self.samples, self.pg = [], []
self.stop = threading.Event()
def run(self):
ids = list(self.containers)
while not self.stop.is_set():
t = time.monotonic()
out = sh(['docker', 'stats', '--no-stream', '--format', '{{.ID}} {{.CPUPerc}} {{.MemUsage}}', *ids], check=False)
for line in out.splitlines():
parts = line.split()
if len(parts) < 2:
continue
try:
cpu = float(parts[1].rstrip('%'))
except ValueError:
continue
mem = parts[2] if len(parts) > 2 else ''
self.samples.append((t, parts[0][:12], cpu, mem))
q = ("select count(*) filter (where state='active'), count(*) "
"from pg_stat_activity where datname='messenger' and pid<>pg_backend_pid()")
pg = self.stack.compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-tAc', q, check=False)
try:
a, total = pg.split('|')
self.pg.append((t, int(a), int(total)))
except ValueError:
pass
def mem_mib(s):
s = s.strip()
for unit, k in (('GiB', 1024), ('MiB', 1), ('KiB', 1 / 1024), ('B', 1 / 1024 / 1024)):
if s.endswith(unit):
try:
return float(s[:-len(unit)]) * k
except ValueError:
return None
return None
def summarize_resources(sampler, containers, env, t_from, t_to):
limits = {'api': float(env['API_CPUS']), 'postgres': float(env['PG_CPUS']),
'lb': float(env['LB_CPUS']), 'redis': float(env['REDIS_CPUS'])}
per = {}
for t, cid, cpu, mem in sampler.samples:
if t_from <= t <= t_to and cid in containers:
per.setdefault(cid, []).append((cpu, mem_mib(mem)))
roles = {}
for cid, vals in per.items():
svc = containers[cid]
cpus = [v[0] for v in vals]
mems = [v[1] for v in vals if v[1] is not None]
roles.setdefault(svc, []).append({'cpu_mean': statistics.mean(cpus), 'cpu_max': max(cpus),
'mem_max_mib': max(mems) if mems else None})
out = {}
for svc, items in roles.items():
lim = limits.get(svc)
mean = statistics.mean(i['cpu_mean'] for i in items)
out[svc] = {
'containers': len(items),
'cpu_pct_mean_per_container': round(mean, 1),
'cpu_pct_max_per_container': round(max(i['cpu_max'] for i in items), 1),
# 100% = весь лимит контейнера; ≥85% считаем насыщением.
'cpu_utilization_of_limit': round(mean / (lim * 100), 3) if lim else None,
'mem_max_mib': round(max((i['mem_max_mib'] or 0) for i in items), 1),
}
pg = [x for x in sampler.pg if t_from <= x[0] <= t_to]
if pg:
out['postgres_connections'] = {'active_max': max(x[1] for x in pg),
'active_mean': round(statistics.mean(x[1] for x in pg), 1),
'total_max': max(x[2] for x in pg)}
return out
def pool_delta(before, after):
acq = empty = dur = 0
max_conns = 0
for cid, a in after.items():
b = before.get(cid, {})
acq += a['acquire_count'] - b.get('acquire_count', 0)
empty += a['empty_acquire_count'] - b.get('empty_acquire_count', 0)
dur += a['acquire_duration_ms'] - b.get('acquire_duration_ms', 0)
max_conns += a['max_conns']
return {'acquires': acq, 'empty_acquire_ratio': round(empty / acq, 4) if acq else None,
'avg_acquire_wait_ms': round(dur / acq, 3) if acq else None, 'pool_max_conns_total': max_conns}
def done_keys(path):
keys = set()
if path.exists():
for line in path.read_text(encoding='utf-8').splitlines():
try:
r = json.loads(line)
keys.add((r['series'], r['config'], r['concurrency'], r['repeat']))
except (ValueError, KeyError):
pass
return keys
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('--experiments', default='experiments/track1.json')
p.add_argument('--series', default='', help='comma-separated series names; default all')
p.add_argument('--output', default='evidence/track1/raw.jsonl')
p.add_argument('--quick', action='store_true', help='levels 4,16,64; 15s; 1 repeat')
p.add_argument('--resume', action='store_true')
p.add_argument('--configs', default='', help='comma-separated config labels to run; default all')
p.add_argument('--levels', default='', help='override levels, e.g. 4,16,64')
p.add_argument('--duration', default='', help='override measured duration, e.g. 15s')
p.add_argument('--warmup', default='', help='override warmup, e.g. 3s')
p.add_argument('--repeats', type=int, default=0, help='override repeats')
p.add_argument('--cleanup', action='store_true', help='docker compose down -v for the bench project at the end')
args = p.parse_args()
exp = json.loads((ROOT / args.experiments).read_text(encoding='utf-8'))
out = ROOT / args.output
out.parent.mkdir(parents=True, exist_ok=True)
skip = done_keys(out) if args.resume else set()
wanted = {s for s in args.series.split(',') if s}
stack = Stack(exp['project'], exp['lb_port'])
probe = subprocess.run(['docker', 'version', '--format', '{{.Server.Version}}'], capture_output=True, text=True)
if probe.returncode != 0:
raise SystemExit('Docker Engine недоступен (docker version: %s). '
'Запустите Docker Desktop и дождитесь "Engine running"; '
'на Windows нужны включённая виртуализация и WSL2.' % (probe.stderr.strip().splitlines() or ['?'])[-1])
sh(['go', 'build', '-o', str(BENCH), './cmd/bench'])
host = {'host_cpus': os.cpu_count(), 'docker': sh(['docker', 'version', '--format', '{{.Server.Version}}'], check=False),
'docker_cpus': sh(['docker', 'info', '--format', '{{.NCPU}}'], check=False),
'docker_mem_bytes': sh(['docker', 'info', '--format', '{{.MemTotal}}'], check=False)}
print('host:', json.dumps(host))
built = False
for series in exp['series']:
if wanted and series['name'] not in wanted:
continue
load = dict(exp['load'])
load['levels'] = series.get('levels', load['levels'])
load['repeats'] = series.get('repeats', load['repeats'])
if args.quick:
load.update(levels=[4, 16, 64] if series['name'] != 'D_improvement' else [32], duration='15s',
warmup='3s', repeats=1)
if args.levels:
load['levels'] = [int(x) for x in args.levels.split(',')]
for k in ('duration', 'warmup', 'repeats'):
if getattr(args, k):
load[k] = getattr(args, k)
only = {c for c in args.configs.split(',') if c}
for cfg in series['configs']:
if only and cfg['label'] not in only:
continue
env = dict(os.environ)
env.update(exp['base_env'])
env.update({k: v for k, v in cfg.items() if k != 'label'})
env.setdefault('API_MEMORY', '256M')
env['LB_PORT'] = str(exp['lb_port'])
env['LB_STATS_PORT'] = str(exp['lb_port'] + 324)
todo = [(lvl, rep) for lvl in load['levels'] for rep in range(load['repeats'])
if (series['name'], cfg['label'], lvl, rep) not in skip]
if not todo:
continue
print('== %s / %s' % (series['name'], cfg['label']), flush=True)
stack.up(env, build=not built)
built = True
n = int(env['API_REPLICAS'])
stack.wait_balanced(n)
stack.pad_redis(exp.get('redis_padding_keys', 0))
containers = stack.containers()
config_env = {k: env[k] for k in ('API_REPLICAS', 'API_CPUS', 'API_MEMORY', 'PG_CPUS', 'LB_CPUS',
'REDIS_CPUS', 'DB_POOL_MAX', 'CACHE_INVALIDATION', 'LB_BALANCE')}
for lvl, rep in todo:
before = stack.pool_stats(containers)
sampler = Sampler(stack, containers)
sampler.start()
t_start = time.monotonic()
warm = float(load['warmup'].rstrip('s'))
dur = float(load['duration'].rstrip('s'))
line = sh([str(BENCH), '-base-url', stack.base, '-concurrency', str(lvl),
'-duration', load['duration'], '-warmup', load['warmup'],
'-write-ratio', str(load['write_ratio']), '-chats', str(load['chats']),
'-label', '%s/%s' % (series['name'], cfg['label'])], timeout=dur + warm + 300)
sampler.stop.set()
sampler.join(timeout=30)
after = stack.pool_stats(containers)
bench = json.loads(line.splitlines()[-1])
rec = {'series': series['name'], 'config': cfg['label'], 'concurrency': lvl, 'repeat': rep,
'load': {k: load[k] for k in ('duration', 'warmup', 'write_ratio', 'chats')},
'env': config_env, 'host': host, 'bench': bench,
'resources': summarize_resources(sampler, containers, env, t_start + warm, t_start + warm + dur),
'pool': pool_delta(before, after)}
with out.open('a', encoding='utf-8') as f:
f.write(json.dumps(rec, ensure_ascii=False) + '\n')
print(' c=%-3d rep=%d ok_rps=%.1f p95=%.1fms err=%.4f' % (
lvl, rep, bench['ok_rps'], bench['latency_ms']['p95'], bench['error_ratio']), flush=True)
if args.cleanup:
# Удаляет ТОЛЬКО bench-проект и его синтетические volumes.
stack.compose('down', '-v')
print('raw results:', out)
if __name__ == '__main__':
sys.exit(main())