Files
mtusi2026lab3/scripts/experiments.py
T

226 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Reproducible real-container experiments. Only synthetic data; no tokens on disk."""
import sys,os,json,time,subprocess,statistics,threading,math,uuid
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor
ROOT=Path(__file__).resolve().parents[1]
os.chdir(ROOT)
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,Failure,check
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
def command(args,env=None):
proc=subprocess.Popen(args,env=env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
waited=0
try:
while True:
try:
output,_=proc.communicate(timeout=15)
break
except subprocess.TimeoutExpired:
waited+=15
print(' Ожидание: '+' '.join(args[:3])+'… ('+str(waited)+' с)',flush=True)
except KeyboardInterrupt:
proc.terminate()
try: proc.communicate(timeout=5)
except subprocess.TimeoutExpired:
proc.kill(); proc.communicate()
raise
if proc.returncode: raise RuntimeError('Command failed: '+str(args)+'\n'+output)
return output
def dc(*args,env=None): return command(['docker','compose',*args],env)
def save(name,data): (OUT/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
def inventory():
ids=dc('ps','-q').split()
return [json.loads(line) for line in command(['docker','inspect','--format','{"id":"{{.Id}}","name":"{{.Name}}","hostname":"{{.Config.Hostname}}","state":"{{.State.Status}}","image":"{{.Config.Image}}","nano_cpus":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},"published_ports":{{json .HostConfig.PortBindings}}}',*ids]).splitlines()]
def up_wait(env=None,replicas=None):
print('Подготовка стенда из локальных образов, ожидание готовности (до 90 с)…',flush=True)
args=['up','-d','--no-build','--pull','never','--wait','--wait-timeout','90']
if replicas is not None: args+=['--scale','api='+str(replicas)]
try: dc(*args,env=env)
except RuntimeError as exc:
# A recreated DB invalidates existing pool connections. Compose may fail
# immediately on an old 'unhealthy' status; restart API pools, then wait.
if 'unhealthy' not in str(exc): raise
dc('restart','api',env=env)
dc(*args,env=env)
time.sleep(3)
def restore(): up_wait()
def sample_stats(stop,rows):
while not stop.is_set():
try:
rows.append({'at':time.time(),'containers':[json.loads(x) for x in dc('stats','--no-stream','--format','{{json .}}').splitlines()]})
except Exception: pass
stop.wait(.5)
def benchmark(duration=10,concurrency=16,action=None):
c=Client(); a,b,e,chat=c.bootstrap()
# Explicit warm-up, excluded from measured interval.
for i in range(50): c.message(a,chat,'warmup-'+str(i))
barrier=threading.Barrier(concurrency+1,action=lambda: started.__setitem__(0,time.monotonic())); lat=[]; errors=[]; observed=set(); confirmed=set(); lock=threading.Lock()
started=[0]; stats=[]; stop=threading.Event()
sampler=threading.Thread(target=sample_stats,args=(stop,stats)); sampler.start()
action_result={}
def run_action():
time.sleep(2)
try: action_result['output']=action(); action_result['ok']=True
except Exception as exc: action_result.update(ok=False,error=str(exc))
action_result['finished_seconds']=time.monotonic()-started[0]
def worker(n):
local=Client(timeout=6); barrier.wait(); index=0
while time.monotonic()-started[0]<duration:
begin=time.monotonic()
try:
response=local.message(a,chat,'load-%s-%s'%(n,index))
# Each worker also reads periodically during failures/updates.
if index%10==0: local.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=1',actor=b,schema='MessagePage')
with lock:
lat.append((time.monotonic()-begin)*1000); observed.add(response['headers']['X-Instance-Id']); confirmed.add(response['json']['id'])
except Failure as exc:
with lock: errors.append({'at_seconds':round(begin-started[0],4),'error':str(exc)})
index+=1
with ThreadPoolExecutor(max_workers=concurrency) as pool:
jobs=[pool.submit(worker,n) for n in range(concurrency)]
started[0]=time.monotonic(); barrier.wait()
thread=threading.Thread(target=run_action) if action else None
if thread: thread.start()
for job in jobs: job.result()
elapsed=time.monotonic()-started[0]
if thread: thread.join()
stop.set(); sampler.join()
lat.sort()
def pct(p): return round(lat[max(0,math.ceil(p*len(lat))-1)],3) if lat else None
result={'duration':duration,'elapsed':round(elapsed,3),'concurrency':concurrency,'success':len(lat),'errors':len(errors),'rps':round(len(lat)/elapsed,3),'p50_ms':pct(.5),'p95_ms':pct(.95),'p99_ms':pct(.99),'instances':sorted(observed),'error_details':errors,'stats':stats,'action':action_result,'warmup_messages':50}
check(bool(lat),'no successes')
if action:
check(action_result.get('ok'),'action failed')
from resilience import history
stored=history(c,b,chat)
check(confirmed.issubset({item['id'] for item in stored}),'confirmed message lost during action')
check(len(stored)==len({item['id'] for item in stored}),'duplicate IDs')
result['confirmed_messages_verified']=len(confirmed)
return result
def correctness():
print('Проверки займут несколько минут. Результаты сохраняются в evidence/.',flush=True)
restore()
print('[1/6] Публичные smoke и проверка структуры',flush=True)
# Preserve raw stdout from unmodified public tests.
(OUT/'smoke.txt').write_text(command(['make','test']))
(OUT/'check.txt').write_text(command(['make','check']))
before=inventory(); save('containers-initial.json',before)
print('[2/6] 24 конкурентных повтора одного сообщения',flush=True)
c=Client(); a,b,e,chat=c.bootstrap(); path='/api/v1/chats/'+chat['id']+'/messages'
key=uuid.uuid4().hex; gate=threading.Barrier(24)
def repeat(i):
local=Client(); gate.wait()
return local.message(a,chat,'parallel-idempotency',key=key,attachments=[] if i%2 else None)
with ThreadPoolExecutor(max_workers=24) as pool: replies=list(pool.map(repeat,range(24)))
check(len({r['json']['id'] for r in replies})==1,'duplicate idempotency effects')
instances=sorted({r['headers']['X-Instance-Id'] for r in replies}); check(len(instances)>=2,'parallel replay missed replicas')
items=c.call('GET',path,actor=b,schema='MessagePage')['json']['items']; check(len(items)==1,'duplicate stored messages')
c.call('POST',path,409,actor=a,body={'text':'different'},headers={'Idempotency-Key':key})
save('idempotency.json',{'concurrent_requests':24,'instances':instances,'distinct_message_ids':1,'stored_messages':len(items),'different_body_status':409,'absent_vs_empty_attachments':'equal'})
# Public test chooses target from the response: rotate serving order until each physical container was stopped.
print('[3/6] Отказ каждой API по очереди (15 с на попытку)',flush=True)
seen=set(); attempts=[]
for attempt in range(8):
restore()
if attempt%2: Client().call('GET','/health/live',schema='Health') # rotate round-robin before target selection
print(' Попытка '+str(attempt+1),flush=True)
output=command(['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh'])
stopped=dc('ps','-a','--format','json')
states=[json.loads(line) for line in stopped.splitlines() if line.strip()]
targets=[r['Name'] for r in states if r['Service']=='api' and r['State']=='exited']
seen.update(targets)
attempts.append({'attempt':attempt+1,'stdout':output,'stopped':targets,'inventory_after':[{'name':r['Name'],'state':r['State'],'id':r['ID']} for r in states]})
save('failover.json',attempts)
if len(seen)>=2: break
check(len(seen)>=2,'did not stop each API replica')
restore()
print('[4/6] Перезапуск стенда с сохранением данных и сессии',flush=True)
(OUT/'persistence.txt').write_text(command(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh']))
save('containers-after-restart.json',inventory())
print('[5/6] Последовательная замена контейнеров под нагрузкой (35 с)',flush=True)
save('rolling.json',benchmark(35,16,lambda:command(['sh','scripts/rolling-update.sh'])))
save('containers-after-rolling.json',inventory())
restore()
print('[6/6] Остановка каждой API под нагрузкой (15 с на реплику)',flush=True)
availability=[]
for item in inventory():
if '-api-' not in item['name']: continue
target=item['hostname']+'-selected'
def action(target=target):
return command(['sh','scripts/stop-one.sh'],dict(os.environ,TARGET_INSTANCE_ID=target))
row=benchmark(15,16,action); row['target_container']=item['name']
availability.append(row); save('availability-load.json',availability)
restore()
print('Correctness, each-replica failure, persistence and rolling: PASS',flush=True)
def configure(n,cpu,memory,pool=8,fast=1,db_cpu=1):
env=dict(os.environ,DB_CPUS=str(db_cpu),API_REPLICAS=str(n),API_CPUS=str(cpu),API_MEMORY=memory,POOL_MAX=str(pool),MESSAGE_ACCESS_FAST=str(fast))
up_wait(env,n)
return env
def scaling():
results=[]
for mode in ('fixed','growing'):
for n in (1,2,4):
cpu=1/n if mode=='fixed' else .5
mem=str(1024//n if mode=='fixed' else 256)+'m'
configure(n,cpu,mem)
for repeat in range(1,4):
row=benchmark(10,32); row.update(series=mode,replicas=n,cpu_per_api=cpu,memory_per_api=mem,repeat=repeat,pool=8)
results.append(row); save('scaling.json',results)
print(mode,n,repeat,row['rps'],row['errors'],flush=True)
sweeps=[]
configure(4,.5,'256m')
for concurrency in (1,4,16,64,128):
row=benchmark(10,concurrency); row.update(replicas=4)
sweeps.append(row); save('saturation.json',sweeps)
print('saturation',concurrency,row['rps'],row['p95_ms'],flush=True)
improvements=[]
for fast in (0,1):
configure(4,.5,'256m',fast=fast)
for repeat in range(1,4):
row=benchmark(10,32); row.update(fast_membership=bool(fast),repeat=repeat)
improvements.append(row); save('improvement.json',improvements)
print('improvement',fast,repeat,row['rps'],flush=True)
configure(2,.5,'256m')
save('containers-final.json',inventory())
def improvement_low():
rows=[]
# Alternate A/B ordering to reduce monotonic database growth/temperature bias.
for repeat in (1,2,3):
for fast in ((0,1) if repeat%2 else (1,0)):
configure(4,.5,'256m',fast=fast)
row=benchmark(10,4); row.update(fast_membership=bool(fast),repeat=repeat)
rows.append(row); save('improvement-low.json',rows)
print('improvement C4',fast,repeat,row['rps'],flush=True)
configure(2,.5,'256m')
def bottleneck():
results=[]
for db_cpu in (.1,1):
for n in (1,2,4):
configure(n,.5,'256m',db_cpu=db_cpu)
for repeat in (1,2):
row=benchmark(10,4)
row.update(db_cpu=db_cpu,replicas=n,repeat=repeat)
results.append(row); save('db-bottleneck.json',results)
print('db',db_cpu,n,repeat,row['rps'],row['p95_ms'],flush=True)
configure(2,.5,'256m')
if __name__=='__main__':
try:
mode=sys.argv[1] if len(sys.argv)>1 else 'correctness'
actions={'correctness':correctness,'scaling':scaling,'bottleneck':bottleneck,'improvement-low':improvement_low}
if mode not in actions: raise SystemExit('Unknown experiment: '+mode)
actions[mode]()
except KeyboardInterrupt:
print('\nПроверка прервана. Для восстановления стенда выполните:\n'
'docker compose up -d --no-build --pull never --wait --wait-timeout 120',flush=True)
raise SystemExit(130) from None