Завершить уровень 4: проверить отказы, поочерёдную замену и ограничить ожидания кэша

This commit is contained in:
2026-10-23 19:09:56 +03:00
parent bdcf00a412
commit d30f418cec
33 changed files with 1962 additions and 40 deletions
+122
View File
@@ -0,0 +1,122 @@
"""Воспроизводимые серии lab3; контейнеры и тома предыдущей лабы не затрагиваются."""
import argparse
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import random
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / 'evidence' / 'research'
OUT.mkdir(parents=True, exist_ok=True)
def command(*args, env=None):
r = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True)
if r.returncode:
raise RuntimeError('Ошибка команды: ' + ' '.join(args[:4]) + '\n' + r.stderr[-2000:])
return r.stdout.strip()
def configure(n, cpu, memory, pool, optimized=1):
env = dict(os.environ, API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=f'{memory}m', POOL_SIZE=str(pool), OPTIMIZED_WRITES=str(optimized), PYTHONUTF8='1')
command('docker', 'compose', 'up', '-d', '--no-deps', '--scale', f'api={n}', 'api', env=env)
command(sys.executable, 'scripts/wait_ready.py', '--replicas', str(n), env=env)
return env
def state():
ids = command('docker', 'compose', 'ps', '-q').splitlines()
return [json.loads(command('docker', 'inspect', '--format', '{{json .State}}', i)) | {'id': i} for i in ids]
def sql(query):
return command('docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
def cpu_counters():
result={}
for cid in command('docker','compose','ps','-q','api').splitlines():
raw=command('docker','exec',cid,'cat','/sys/fs/cgroup/cpu.stat')
result[cid]={line.split()[0]:int(line.split()[1]) for line in raw.splitlines()}
return result
def benchmark(label, n, cpu, memory, pool, concurrency=32, duration=20, rolling=False, optimized=1):
configure(n, cpu, memory, pool, optimized)
started = datetime.now(timezone.utc).isoformat()
before = sql('SELECT count(*) FROM messages')
cpu_before=cpu_counters()
before_state = state() if rolling else None
args = ['docker', 'run', '--rm', '--name', 'lyakhonov-lab3-load', '--network', 'lyakhonov-lab3_default', '--cpus', '2', '--memory', '512m',
'-v', f'{ROOT / "student-tests"}:/work/student-tests:ro', '-v', f'{ROOT / "tests"}:/work/tests:ro',
'-v', f'{ROOT / "lab.json"}:/work/lab.json:ro', '-v', f'{ROOT / "contracts"}:/work/contracts:ro',
'-w', '/work', 'lyakhonov-messenger:lab3', 'python', 'student-tests/benchmark.py', '--concurrency', str(concurrency), '--duration', str(duration)]
proc = subprocess.Popen(args, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
samples, waits, actions = [], [], []
timer = time.monotonic()
rolling_ids = command('docker', 'compose', 'ps', '-q', 'api').splitlines() if rolling else []
while proc.poll() is None:
raw = command('docker', 'stats', '--no-stream', '--format', '{{json .}}')
samples.append({'seconds': round(time.monotonic()-timer, 3), 'containers': [json.loads(line) for line in raw.splitlines() if 'lyakhonov-lab3' in line]})
waits.append(sql("SELECT coalesce(wait_event_type,'CPU/active')||':'||coalesce(wait_event,'running')||':'||count(*) FROM pg_stat_activity WHERE datname='messenger' AND backend_type='client backend' GROUP BY 1=0,wait_event_type,wait_event"))
# Поочерёдная замена каждого контейнера: старый останавливается, новый создаётся с тем же номером сервиса.
if rolling and rolling_ids and time.monotonic()-timer > 10 + len(actions)*10:
old = rolling_ids.pop(0)
step = time.monotonic()
command('docker', 'stop', '--time', '10', old)
command('docker', 'rm', old)
configure(n, cpu, memory, pool)
actions.append({'old_id': old, 'start_seconds': step-timer, 'finished_seconds': time.monotonic()-timer})
stdout, stderr = proc.communicate()
if proc.returncode:
raise RuntimeError(stderr)
result = json.loads(stdout)
cpu_after=cpu_counters()
result.update({'label': label, 'started_utc': started, 'replicas': n, 'cpu_per_replica': cpu, 'memory_mib_per_replica': memory, 'pool_per_replica': pool,
'database_messages_before': int(before), 'database_messages_after': int(sql('SELECT count(*) FROM messages')),
'samples': samples, 'database_waits': waits, 'rolling_actions': actions, 'before_state': before_state, 'after_state': state() if rolling else None})
result['api_cgroup_cpu_delta']={cid:{k:v-cpu_before[cid].get(k,0) for k,v in counters.items()} for cid,counters in cpu_after.items() if cid in cpu_before}
result['optimized_writes_requested']=optimized
result['application_image_id']=command('docker','image','inspect','lyakhonov-messenger:lab3','--format','{{.Id}}')
(OUT / (label+'.json')).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n', encoding='utf-8')
print(label, 'RPS=', round(result['successful_rps'], 2), 'errors=', result['failed_requests'], flush=True)
return result
def main():
p = argparse.ArgumentParser()
p.add_argument('--phase', choices=['pilot', 'matrix', 'saturation', 'pool', 'rolling', 'optimize'], required=True)
p.add_argument('--only', action='append', default=[], help='Повторить только названные точки выбранной серии')
args = p.parse_args()
if args.phase == 'pilot':
benchmark('pilot', 2, 1, 512, 10, duration=10)
elif args.phase == 'matrix':
jobs = [(mode, n, rep) for mode in ['fixed', 'growth'] for n in [1, 2, 4] for rep in range(1, 4)]
random.Random(2301).shuffle(jobs)
for mode, n, rep in jobs:
label=f'{mode}-{n}-{rep}'
if not args.only or label in args.only:
benchmark(label, n, 2/n if mode=='fixed' else .5, 1024//n if mode=='fixed' else 256, 10)
elif args.phase == 'saturation':
for concurrency in [4, 16, 64, 128]:
for rep in range(1, 3):
benchmark(f'saturation-{concurrency}-{rep}', 4, .5, 256, 10, concurrency)
elif args.phase == 'pool':
for rep in range(1, 4):
for pool in ([1,10] if rep%2 else [10,1]):
benchmark(f'pool-{pool}-{rep}', 2, 1, 512, pool, 64)
elif args.phase == 'rolling':
benchmark('rolling', 2, .5, 256, 10, 16, 45, True)
elif args.phase == 'optimize':
for rep in range(1,4):
for mode in ([0,1] if rep%2 else [1,0]):
benchmark(f'optimize-{mode}-{rep}',2,1,512,10,32,optimized=mode)
configure(2, .5, 256, 10)
if __name__ == '__main__':
main()