Завершить уровень 4: проверить отказы, поочерёдную замену и ограничить ожидания кэша
This commit is contained in:
@@ -0,0 +1,122 @@
|
||||
"""Воспроизводимые серии lab3; контейнеры и тома предыдущей лабы не затрагиваются."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT / 'evidence' / 'research'
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def command(*args, env=None):
|
||||
r = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True)
|
||||
if r.returncode:
|
||||
raise RuntimeError('Ошибка команды: ' + ' '.join(args[:4]) + '\n' + r.stderr[-2000:])
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
def configure(n, cpu, memory, pool, optimized=1):
|
||||
env = dict(os.environ, API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=f'{memory}m', POOL_SIZE=str(pool), OPTIMIZED_WRITES=str(optimized), PYTHONUTF8='1')
|
||||
command('docker', 'compose', 'up', '-d', '--no-deps', '--scale', f'api={n}', 'api', env=env)
|
||||
command(sys.executable, 'scripts/wait_ready.py', '--replicas', str(n), env=env)
|
||||
return env
|
||||
|
||||
|
||||
def state():
|
||||
ids = command('docker', 'compose', 'ps', '-q').splitlines()
|
||||
return [json.loads(command('docker', 'inspect', '--format', '{{json .State}}', i)) | {'id': i} for i in ids]
|
||||
|
||||
|
||||
def sql(query):
|
||||
return command('docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
||||
|
||||
|
||||
def cpu_counters():
|
||||
result={}
|
||||
for cid in command('docker','compose','ps','-q','api').splitlines():
|
||||
raw=command('docker','exec',cid,'cat','/sys/fs/cgroup/cpu.stat')
|
||||
result[cid]={line.split()[0]:int(line.split()[1]) for line in raw.splitlines()}
|
||||
return result
|
||||
|
||||
|
||||
def benchmark(label, n, cpu, memory, pool, concurrency=32, duration=20, rolling=False, optimized=1):
|
||||
configure(n, cpu, memory, pool, optimized)
|
||||
started = datetime.now(timezone.utc).isoformat()
|
||||
before = sql('SELECT count(*) FROM messages')
|
||||
cpu_before=cpu_counters()
|
||||
before_state = state() if rolling else None
|
||||
args = ['docker', 'run', '--rm', '--name', 'lyakhonov-lab3-load', '--network', 'lyakhonov-lab3_default', '--cpus', '2', '--memory', '512m',
|
||||
'-v', f'{ROOT / "student-tests"}:/work/student-tests:ro', '-v', f'{ROOT / "tests"}:/work/tests:ro',
|
||||
'-v', f'{ROOT / "lab.json"}:/work/lab.json:ro', '-v', f'{ROOT / "contracts"}:/work/contracts:ro',
|
||||
'-w', '/work', 'lyakhonov-messenger:lab3', 'python', 'student-tests/benchmark.py', '--concurrency', str(concurrency), '--duration', str(duration)]
|
||||
proc = subprocess.Popen(args, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
samples, waits, actions = [], [], []
|
||||
timer = time.monotonic()
|
||||
rolling_ids = command('docker', 'compose', 'ps', '-q', 'api').splitlines() if rolling else []
|
||||
while proc.poll() is None:
|
||||
raw = command('docker', 'stats', '--no-stream', '--format', '{{json .}}')
|
||||
samples.append({'seconds': round(time.monotonic()-timer, 3), 'containers': [json.loads(line) for line in raw.splitlines() if 'lyakhonov-lab3' in line]})
|
||||
waits.append(sql("SELECT coalesce(wait_event_type,'CPU/active')||':'||coalesce(wait_event,'running')||':'||count(*) FROM pg_stat_activity WHERE datname='messenger' AND backend_type='client backend' GROUP BY 1=0,wait_event_type,wait_event"))
|
||||
# Поочерёдная замена каждого контейнера: старый останавливается, новый создаётся с тем же номером сервиса.
|
||||
if rolling and rolling_ids and time.monotonic()-timer > 10 + len(actions)*10:
|
||||
old = rolling_ids.pop(0)
|
||||
step = time.monotonic()
|
||||
command('docker', 'stop', '--time', '10', old)
|
||||
command('docker', 'rm', old)
|
||||
configure(n, cpu, memory, pool)
|
||||
actions.append({'old_id': old, 'start_seconds': step-timer, 'finished_seconds': time.monotonic()-timer})
|
||||
stdout, stderr = proc.communicate()
|
||||
if proc.returncode:
|
||||
raise RuntimeError(stderr)
|
||||
result = json.loads(stdout)
|
||||
cpu_after=cpu_counters()
|
||||
result.update({'label': label, 'started_utc': started, 'replicas': n, 'cpu_per_replica': cpu, 'memory_mib_per_replica': memory, 'pool_per_replica': pool,
|
||||
'database_messages_before': int(before), 'database_messages_after': int(sql('SELECT count(*) FROM messages')),
|
||||
'samples': samples, 'database_waits': waits, 'rolling_actions': actions, 'before_state': before_state, 'after_state': state() if rolling else None})
|
||||
result['api_cgroup_cpu_delta']={cid:{k:v-cpu_before[cid].get(k,0) for k,v in counters.items()} for cid,counters in cpu_after.items() if cid in cpu_before}
|
||||
result['optimized_writes_requested']=optimized
|
||||
result['application_image_id']=command('docker','image','inspect','lyakhonov-messenger:lab3','--format','{{.Id}}')
|
||||
(OUT / (label+'.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
|
||||
print(label, 'RPS=', round(result['successful_rps'], 2), 'errors=', result['failed_requests'], flush=True)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--phase', choices=['pilot', 'matrix', 'saturation', 'pool', 'rolling', 'optimize'], required=True)
|
||||
p.add_argument('--only', action='append', default=[], help='Повторить только названные точки выбранной серии')
|
||||
args = p.parse_args()
|
||||
if args.phase == 'pilot':
|
||||
benchmark('pilot', 2, 1, 512, 10, duration=10)
|
||||
elif args.phase == 'matrix':
|
||||
jobs = [(mode, n, rep) for mode in ['fixed', 'growth'] for n in [1, 2, 4] for rep in range(1, 4)]
|
||||
random.Random(2301).shuffle(jobs)
|
||||
for mode, n, rep in jobs:
|
||||
label=f'{mode}-{n}-{rep}'
|
||||
if not args.only or label in args.only:
|
||||
benchmark(label, n, 2/n if mode=='fixed' else .5, 1024//n if mode=='fixed' else 256, 10)
|
||||
elif args.phase == 'saturation':
|
||||
for concurrency in [4, 16, 64, 128]:
|
||||
for rep in range(1, 3):
|
||||
benchmark(f'saturation-{concurrency}-{rep}', 4, .5, 256, 10, concurrency)
|
||||
elif args.phase == 'pool':
|
||||
for rep in range(1, 4):
|
||||
for pool in ([1,10] if rep%2 else [10,1]):
|
||||
benchmark(f'pool-{pool}-{rep}', 2, 1, 512, pool, 64)
|
||||
elif args.phase == 'rolling':
|
||||
benchmark('rolling', 2, .5, 256, 10, 16, 45, True)
|
||||
elif args.phase == 'optimize':
|
||||
for rep in range(1,4):
|
||||
for mode in ([0,1] if rep%2 else [1,0]):
|
||||
benchmark(f'optimize-{mode}-{rep}',2,1,512,10,32,optimized=mode)
|
||||
configure(2, .5, 256, 10)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user