226 lines
12 KiB
Python
226 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""Reproducible real-container experiments. Only synthetic data; no tokens on disk."""
|
||
import sys,os,json,time,subprocess,statistics,threading,math,uuid
|
||
from pathlib import Path
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
ROOT=Path(__file__).resolve().parents[1]
|
||
os.chdir(ROOT)
|
||
sys.path.insert(0,str(ROOT/'tests'))
|
||
from client import Client,Failure,check
|
||
OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True)
|
||
|
||
def command(args,env=None):
|
||
proc=subprocess.Popen(args,env=env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
||
waited=0
|
||
try:
|
||
while True:
|
||
try:
|
||
output,_=proc.communicate(timeout=15)
|
||
break
|
||
except subprocess.TimeoutExpired:
|
||
waited+=15
|
||
print(' Ожидание: '+' '.join(args[:3])+'… ('+str(waited)+' с)',flush=True)
|
||
except KeyboardInterrupt:
|
||
proc.terminate()
|
||
try: proc.communicate(timeout=5)
|
||
except subprocess.TimeoutExpired:
|
||
proc.kill(); proc.communicate()
|
||
raise
|
||
if proc.returncode: raise RuntimeError('Command failed: '+str(args)+'\n'+output)
|
||
return output
|
||
|
||
def dc(*args,env=None): return command(['docker','compose',*args],env)
|
||
def save(name,data): (OUT/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n')
|
||
def inventory():
|
||
ids=dc('ps','-q').split()
|
||
return [json.loads(line) for line in command(['docker','inspect','--format','{"id":"{{.Id}}","name":"{{.Name}}","hostname":"{{.Config.Hostname}}","state":"{{.State.Status}}","image":"{{.Config.Image}}","nano_cpus":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},"published_ports":{{json .HostConfig.PortBindings}}}',*ids]).splitlines()]
|
||
def up_wait(env=None,replicas=None):
|
||
print('Подготовка стенда из локальных образов, ожидание готовности (до 90 с)…',flush=True)
|
||
args=['up','-d','--no-build','--pull','never','--wait','--wait-timeout','90']
|
||
if replicas is not None: args+=['--scale','api='+str(replicas)]
|
||
try: dc(*args,env=env)
|
||
except RuntimeError as exc:
|
||
# A recreated DB invalidates existing pool connections. Compose may fail
|
||
# immediately on an old 'unhealthy' status; restart API pools, then wait.
|
||
if 'unhealthy' not in str(exc): raise
|
||
dc('restart','api',env=env)
|
||
dc(*args,env=env)
|
||
time.sleep(3)
|
||
|
||
def restore(): up_wait()
|
||
def sample_stats(stop,rows):
|
||
while not stop.is_set():
|
||
try:
|
||
rows.append({'at':time.time(),'containers':[json.loads(x) for x in dc('stats','--no-stream','--format','{{json .}}').splitlines()]})
|
||
except Exception: pass
|
||
stop.wait(.5)
|
||
def benchmark(duration=10,concurrency=16,action=None):
|
||
c=Client(); a,b,e,chat=c.bootstrap()
|
||
# Explicit warm-up, excluded from measured interval.
|
||
for i in range(50): c.message(a,chat,'warmup-'+str(i))
|
||
barrier=threading.Barrier(concurrency+1,action=lambda: started.__setitem__(0,time.monotonic())); lat=[]; errors=[]; observed=set(); confirmed=set(); lock=threading.Lock()
|
||
started=[0]; stats=[]; stop=threading.Event()
|
||
sampler=threading.Thread(target=sample_stats,args=(stop,stats)); sampler.start()
|
||
action_result={}
|
||
def run_action():
|
||
time.sleep(2)
|
||
try: action_result['output']=action(); action_result['ok']=True
|
||
except Exception as exc: action_result.update(ok=False,error=str(exc))
|
||
action_result['finished_seconds']=time.monotonic()-started[0]
|
||
def worker(n):
|
||
local=Client(timeout=6); barrier.wait(); index=0
|
||
while time.monotonic()-started[0]<duration:
|
||
begin=time.monotonic()
|
||
try:
|
||
response=local.message(a,chat,'load-%s-%s'%(n,index))
|
||
# Each worker also reads periodically during failures/updates.
|
||
if index%10==0: local.call('GET','/api/v1/chats/'+chat['id']+'/messages?limit=1',actor=b,schema='MessagePage')
|
||
with lock:
|
||
lat.append((time.monotonic()-begin)*1000); observed.add(response['headers']['X-Instance-Id']); confirmed.add(response['json']['id'])
|
||
except Failure as exc:
|
||
with lock: errors.append({'at_seconds':round(begin-started[0],4),'error':str(exc)})
|
||
index+=1
|
||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||
jobs=[pool.submit(worker,n) for n in range(concurrency)]
|
||
started[0]=time.monotonic(); barrier.wait()
|
||
thread=threading.Thread(target=run_action) if action else None
|
||
if thread: thread.start()
|
||
for job in jobs: job.result()
|
||
elapsed=time.monotonic()-started[0]
|
||
if thread: thread.join()
|
||
stop.set(); sampler.join()
|
||
lat.sort()
|
||
def pct(p): return round(lat[max(0,math.ceil(p*len(lat))-1)],3) if lat else None
|
||
result={'duration':duration,'elapsed':round(elapsed,3),'concurrency':concurrency,'success':len(lat),'errors':len(errors),'rps':round(len(lat)/elapsed,3),'p50_ms':pct(.5),'p95_ms':pct(.95),'p99_ms':pct(.99),'instances':sorted(observed),'error_details':errors,'stats':stats,'action':action_result,'warmup_messages':50}
|
||
check(bool(lat),'no successes')
|
||
if action:
|
||
check(action_result.get('ok'),'action failed')
|
||
from resilience import history
|
||
stored=history(c,b,chat)
|
||
check(confirmed.issubset({item['id'] for item in stored}),'confirmed message lost during action')
|
||
check(len(stored)==len({item['id'] for item in stored}),'duplicate IDs')
|
||
result['confirmed_messages_verified']=len(confirmed)
|
||
return result
|
||
|
||
def correctness():
|
||
print('Проверки займут несколько минут. Результаты сохраняются в evidence/.',flush=True)
|
||
restore()
|
||
print('[1/6] Публичные smoke и проверка структуры',flush=True)
|
||
# Preserve raw stdout from unmodified public tests.
|
||
(OUT/'smoke.txt').write_text(command(['make','test']))
|
||
(OUT/'check.txt').write_text(command(['make','check']))
|
||
before=inventory(); save('containers-initial.json',before)
|
||
print('[2/6] 24 конкурентных повтора одного сообщения',flush=True)
|
||
c=Client(); a,b,e,chat=c.bootstrap(); path='/api/v1/chats/'+chat['id']+'/messages'
|
||
key=uuid.uuid4().hex; gate=threading.Barrier(24)
|
||
def repeat(i):
|
||
local=Client(); gate.wait()
|
||
return local.message(a,chat,'parallel-idempotency',key=key,attachments=[] if i%2 else None)
|
||
with ThreadPoolExecutor(max_workers=24) as pool: replies=list(pool.map(repeat,range(24)))
|
||
check(len({r['json']['id'] for r in replies})==1,'duplicate idempotency effects')
|
||
instances=sorted({r['headers']['X-Instance-Id'] for r in replies}); check(len(instances)>=2,'parallel replay missed replicas')
|
||
items=c.call('GET',path,actor=b,schema='MessagePage')['json']['items']; check(len(items)==1,'duplicate stored messages')
|
||
c.call('POST',path,409,actor=a,body={'text':'different'},headers={'Idempotency-Key':key})
|
||
save('idempotency.json',{'concurrent_requests':24,'instances':instances,'distinct_message_ids':1,'stored_messages':len(items),'different_body_status':409,'absent_vs_empty_attachments':'equal'})
|
||
# Public test chooses target from the response: rotate serving order until each physical container was stopped.
|
||
print('[3/6] Отказ каждой API по очереди (15 с на попытку)',flush=True)
|
||
seen=set(); attempts=[]
|
||
for attempt in range(8):
|
||
restore()
|
||
if attempt%2: Client().call('GET','/health/live',schema='Health') # rotate round-robin before target selection
|
||
print(' Попытка '+str(attempt+1),flush=True)
|
||
output=command(['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh'])
|
||
stopped=dc('ps','-a','--format','json')
|
||
states=[json.loads(line) for line in stopped.splitlines() if line.strip()]
|
||
targets=[r['Name'] for r in states if r['Service']=='api' and r['State']=='exited']
|
||
seen.update(targets)
|
||
attempts.append({'attempt':attempt+1,'stdout':output,'stopped':targets,'inventory_after':[{'name':r['Name'],'state':r['State'],'id':r['ID']} for r in states]})
|
||
save('failover.json',attempts)
|
||
if len(seen)>=2: break
|
||
check(len(seen)>=2,'did not stop each API replica')
|
||
restore()
|
||
print('[4/6] Перезапуск стенда с сохранением данных и сессии',flush=True)
|
||
(OUT/'persistence.txt').write_text(command(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh']))
|
||
save('containers-after-restart.json',inventory())
|
||
print('[5/6] Последовательная замена контейнеров под нагрузкой (35 с)',flush=True)
|
||
save('rolling.json',benchmark(35,16,lambda:command(['sh','scripts/rolling-update.sh'])))
|
||
save('containers-after-rolling.json',inventory())
|
||
restore()
|
||
print('[6/6] Остановка каждой API под нагрузкой (15 с на реплику)',flush=True)
|
||
availability=[]
|
||
for item in inventory():
|
||
if '-api-' not in item['name']: continue
|
||
target=item['hostname']+'-selected'
|
||
def action(target=target):
|
||
return command(['sh','scripts/stop-one.sh'],dict(os.environ,TARGET_INSTANCE_ID=target))
|
||
row=benchmark(15,16,action); row['target_container']=item['name']
|
||
availability.append(row); save('availability-load.json',availability)
|
||
restore()
|
||
print('Correctness, each-replica failure, persistence and rolling: PASS',flush=True)
|
||
|
||
def configure(n,cpu,memory,pool=8,fast=1,db_cpu=1):
|
||
env=dict(os.environ,DB_CPUS=str(db_cpu),API_REPLICAS=str(n),API_CPUS=str(cpu),API_MEMORY=memory,POOL_MAX=str(pool),MESSAGE_ACCESS_FAST=str(fast))
|
||
up_wait(env,n)
|
||
return env
|
||
|
||
def scaling():
|
||
results=[]
|
||
for mode in ('fixed','growing'):
|
||
for n in (1,2,4):
|
||
cpu=1/n if mode=='fixed' else .5
|
||
mem=str(1024//n if mode=='fixed' else 256)+'m'
|
||
configure(n,cpu,mem)
|
||
for repeat in range(1,4):
|
||
row=benchmark(10,32); row.update(series=mode,replicas=n,cpu_per_api=cpu,memory_per_api=mem,repeat=repeat,pool=8)
|
||
results.append(row); save('scaling.json',results)
|
||
print(mode,n,repeat,row['rps'],row['errors'],flush=True)
|
||
sweeps=[]
|
||
configure(4,.5,'256m')
|
||
for concurrency in (1,4,16,64,128):
|
||
row=benchmark(10,concurrency); row.update(replicas=4)
|
||
sweeps.append(row); save('saturation.json',sweeps)
|
||
print('saturation',concurrency,row['rps'],row['p95_ms'],flush=True)
|
||
improvements=[]
|
||
for fast in (0,1):
|
||
configure(4,.5,'256m',fast=fast)
|
||
for repeat in range(1,4):
|
||
row=benchmark(10,32); row.update(fast_membership=bool(fast),repeat=repeat)
|
||
improvements.append(row); save('improvement.json',improvements)
|
||
print('improvement',fast,repeat,row['rps'],flush=True)
|
||
configure(2,.5,'256m')
|
||
save('containers-final.json',inventory())
|
||
|
||
def improvement_low():
|
||
rows=[]
|
||
# Alternate A/B ordering to reduce monotonic database growth/temperature bias.
|
||
for repeat in (1,2,3):
|
||
for fast in ((0,1) if repeat%2 else (1,0)):
|
||
configure(4,.5,'256m',fast=fast)
|
||
row=benchmark(10,4); row.update(fast_membership=bool(fast),repeat=repeat)
|
||
rows.append(row); save('improvement-low.json',rows)
|
||
print('improvement C4',fast,repeat,row['rps'],flush=True)
|
||
configure(2,.5,'256m')
|
||
|
||
def bottleneck():
|
||
results=[]
|
||
for db_cpu in (.1,1):
|
||
for n in (1,2,4):
|
||
configure(n,.5,'256m',db_cpu=db_cpu)
|
||
for repeat in (1,2):
|
||
row=benchmark(10,4)
|
||
row.update(db_cpu=db_cpu,replicas=n,repeat=repeat)
|
||
results.append(row); save('db-bottleneck.json',results)
|
||
print('db',db_cpu,n,repeat,row['rps'],row['p95_ms'],flush=True)
|
||
configure(2,.5,'256m')
|
||
|
||
if __name__=='__main__':
|
||
try:
|
||
mode=sys.argv[1] if len(sys.argv)>1 else 'correctness'
|
||
actions={'correctness':correctness,'scaling':scaling,'bottleneck':bottleneck,'improvement-low':improvement_low}
|
||
if mode not in actions: raise SystemExit('Unknown experiment: '+mode)
|
||
actions[mode]()
|
||
except KeyboardInterrupt:
|
||
print('\nПроверка прервана. Для восстановления стенда выполните:\n'
|
||
'docker compose up -d --no-build --pull never --wait --wait-timeout 120',flush=True)
|
||
raise SystemExit(130) from None
|