#!/usr/bin/env python3 """Reproducible real-container experiments. Only synthetic data; no tokens on disk.""" import sys,os,json,time,subprocess,statistics,threading,math,uuid from pathlib import Path from concurrent.futures import ThreadPoolExecutor ROOT=Path(__file__).resolve().parents[1] os.chdir(ROOT) sys.path.insert(0,str(ROOT/'tests')) from client import Client,Failure,check OUT=ROOT/'evidence'; OUT.mkdir(exist_ok=True) def command(args,env=None): proc=subprocess.Popen(args,env=env,text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) waited=0 try: while True: try: output,_=proc.communicate(timeout=15) break except subprocess.TimeoutExpired: waited+=15 print(' Ожидание: '+' '.join(args[:3])+'… ('+str(waited)+' с)',flush=True) except KeyboardInterrupt: proc.terminate() try: proc.communicate(timeout=5) except subprocess.TimeoutExpired: proc.kill(); proc.communicate() raise if proc.returncode: raise RuntimeError('Command failed: '+str(args)+'\n'+output) return output def dc(*args,env=None): return command(['docker','compose',*args],env) def save(name,data): (OUT/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n') def inventory(): ids=dc('ps','-q').split() return [json.loads(line) for line in command(['docker','inspect','--format','{"id":"{{.Id}}","name":"{{.Name}}","hostname":"{{.Config.Hostname}}","state":"{{.State.Status}}","image":"{{.Config.Image}}","nano_cpus":{{.HostConfig.NanoCpus}},"memory":{{.HostConfig.Memory}},"published_ports":{{json .HostConfig.PortBindings}}}',*ids]).splitlines()] def up_wait(env=None,replicas=None): print('Подготовка стенда из локальных образов, ожидание готовности (до 90 с)…',flush=True) args=['up','-d','--no-build','--pull','never','--wait','--wait-timeout','90'] if replicas is not None: args+=['--scale','api='+str(replicas)] try: dc(*args,env=env) except RuntimeError as exc: # A recreated DB invalidates existing pool connections. Compose may fail # immediately on an old 'unhealthy' status; restart API pools, then wait. if 'unhealthy' not in str(exc): raise dc('restart','api',env=env) dc(*args,env=env) time.sleep(3) def restore(): up_wait() def sample_stats(stop,rows): while not stop.is_set(): try: rows.append({'at':time.time(),'containers':[json.loads(x) for x in dc('stats','--no-stream','--format','{{json .}}').splitlines()]}) except Exception: pass stop.wait(.5) def benchmark(duration=10,concurrency=16,action=None): c=Client(); a,b,e,chat=c.bootstrap() # Explicit warm-up, excluded from measured interval. for i in range(50): c.message(a,chat,'warmup-'+str(i)) barrier=threading.Barrier(concurrency+1,action=lambda: started.__setitem__(0,time.monotonic())); lat=[]; errors=[]; observed=set(); confirmed=set(); lock=threading.Lock() started=[0]; stats=[]; stop=threading.Event() sampler=threading.Thread(target=sample_stats,args=(stop,stats)); sampler.start() action_result={} def run_action(): time.sleep(2) try: action_result['output']=action(); action_result['ok']=True except Exception as exc: action_result.update(ok=False,error=str(exc)) action_result['finished_seconds']=time.monotonic()-started[0] def worker(n): local=Client(timeout=6); barrier.wait(); index=0 while time.monotonic()-started[0]=2,'parallel replay missed replicas') items=c.call('GET',path,actor=b,schema='MessagePage')['json']['items']; check(len(items)==1,'duplicate stored messages') c.call('POST',path,409,actor=a,body={'text':'different'},headers={'Idempotency-Key':key}) save('idempotency.json',{'concurrent_requests':24,'instances':instances,'distinct_message_ids':1,'stored_messages':len(items),'different_body_status':409,'absent_vs_empty_attachments':'equal'}) # Public test chooses target from the response: rotate serving order until each physical container was stopped. print('[3/6] Отказ каждой API по очереди (15 с на попытку)',flush=True) seen=set(); attempts=[] for attempt in range(8): restore() if attempt%2: Client().call('GET','/health/live',schema='Health') # rotate round-robin before target selection print(' Попытка '+str(attempt+1),flush=True) output=command(['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh']) stopped=dc('ps','-a','--format','json') states=[json.loads(line) for line in stopped.splitlines() if line.strip()] targets=[r['Name'] for r in states if r['Service']=='api' and r['State']=='exited'] seen.update(targets) attempts.append({'attempt':attempt+1,'stdout':output,'stopped':targets,'inventory_after':[{'name':r['Name'],'state':r['State'],'id':r['ID']} for r in states]}) save('failover.json',attempts) if len(seen)>=2: break check(len(seen)>=2,'did not stop each API replica') restore() print('[4/6] Перезапуск стенда с сохранением данных и сессии',flush=True) (OUT/'persistence.txt').write_text(command(['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh'])) save('containers-after-restart.json',inventory()) print('[5/6] Последовательная замена контейнеров под нагрузкой (35 с)',flush=True) save('rolling.json',benchmark(35,16,lambda:command(['sh','scripts/rolling-update.sh']))) save('containers-after-rolling.json',inventory()) restore() print('[6/6] Остановка каждой API под нагрузкой (15 с на реплику)',flush=True) availability=[] for item in inventory(): if '-api-' not in item['name']: continue target=item['hostname']+'-selected' def action(target=target): return command(['sh','scripts/stop-one.sh'],dict(os.environ,TARGET_INSTANCE_ID=target)) row=benchmark(15,16,action); row['target_container']=item['name'] availability.append(row); save('availability-load.json',availability) restore() print('Correctness, each-replica failure, persistence and rolling: PASS',flush=True) def configure(n,cpu,memory,pool=8,fast=1,db_cpu=1): env=dict(os.environ,DB_CPUS=str(db_cpu),API_REPLICAS=str(n),API_CPUS=str(cpu),API_MEMORY=memory,POOL_MAX=str(pool),MESSAGE_ACCESS_FAST=str(fast)) up_wait(env,n) return env def scaling(): results=[] for mode in ('fixed','growing'): for n in (1,2,4): cpu=1/n if mode=='fixed' else .5 mem=str(1024//n if mode=='fixed' else 256)+'m' configure(n,cpu,mem) for repeat in range(1,4): row=benchmark(10,32); row.update(series=mode,replicas=n,cpu_per_api=cpu,memory_per_api=mem,repeat=repeat,pool=8) results.append(row); save('scaling.json',results) print(mode,n,repeat,row['rps'],row['errors'],flush=True) sweeps=[] configure(4,.5,'256m') for concurrency in (1,4,16,64,128): row=benchmark(10,concurrency); row.update(replicas=4) sweeps.append(row); save('saturation.json',sweeps) print('saturation',concurrency,row['rps'],row['p95_ms'],flush=True) improvements=[] for fast in (0,1): configure(4,.5,'256m',fast=fast) for repeat in range(1,4): row=benchmark(10,32); row.update(fast_membership=bool(fast),repeat=repeat) improvements.append(row); save('improvement.json',improvements) print('improvement',fast,repeat,row['rps'],flush=True) configure(2,.5,'256m') save('containers-final.json',inventory()) def improvement_low(): rows=[] # Alternate A/B ordering to reduce monotonic database growth/temperature bias. for repeat in (1,2,3): for fast in ((0,1) if repeat%2 else (1,0)): configure(4,.5,'256m',fast=fast) row=benchmark(10,4); row.update(fast_membership=bool(fast),repeat=repeat) rows.append(row); save('improvement-low.json',rows) print('improvement C4',fast,repeat,row['rps'],flush=True) configure(2,.5,'256m') def bottleneck(): results=[] for db_cpu in (.1,1): for n in (1,2,4): configure(n,.5,'256m',db_cpu=db_cpu) for repeat in (1,2): row=benchmark(10,4) row.update(db_cpu=db_cpu,replicas=n,repeat=repeat) results.append(row); save('db-bottleneck.json',results) print('db',db_cpu,n,repeat,row['rps'],row['p95_ms'],flush=True) configure(2,.5,'256m') if __name__=='__main__': try: mode=sys.argv[1] if len(sys.argv)>1 else 'correctness' actions={'correctness':correctness,'scaling':scaling,'bottleneck':bottleneck,'improvement-low':improvement_low} if mode not in actions: raise SystemExit('Unknown experiment: '+mode) actions[mode]() except KeyboardInterrupt: print('\nПроверка прервана. Для восстановления стенда выполните:\n' 'docker compose up -d --no-build --pull never --wait --wait-timeout 120',flush=True) raise SystemExit(130) from None