"""Grade-4 evidence: shared idempotency, each replica failure, rolling load.""" import concurrent.futures import json import os from pathlib import Path import subprocess import sys import threading import time import uuid ROOT = Path(__file__).resolve().parents[1] os.chdir(ROOT) sys.path.insert(0,str(ROOT/'tests')) sys.path.insert(0,str(ROOT/'scripts')) from client import Client, Failure from resilience import history from rolling import command, ready base = os.environ.get('BASE_URL','http://localhost:8080') ready() c = Client(base) a,b,_,chat = c.bootstrap() key = uuid.uuid4().hex barrier = threading.Barrier(16) def duplicate(_): barrier.wait() return Client(base).message(a,chat,'Одна запись при гонке',key=key) with concurrent.futures.ThreadPoolExecutor(16) as pool: replies = list(pool.map(duplicate,range(16))) assert len({r['json']['id'] for r in replies})==1 instances = {r['headers']['X-Instance-Id'] for r in replies} assert len(instances)==2 assert len(history(c,b,chat))==1 # Auth and conflict are still checked on replay. c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=a,body={'text':'different'},headers={'Idempotency-Key':key}) result={'parallel_idempotency':{'requests':16,'instances':sorted(instances),'messages':1}, 'failures':[]} def workload(action): stop = threading.Event() started=time.monotonic() confirmed=[] times=[] errors=[] latencies=[] lock=threading.Lock() def worker(n): local=Client(base,timeout=2) pending=uuid.uuid4().hex while not stop.is_set(): t=time.monotonic() try: msg=local.message(a,chat,pending,key=pending)['json'] local.call('GET','/api/v1/users/me',actor=a,schema='User') with lock: confirmed.append(msg); times.append(time.monotonic()); latencies.append(time.monotonic()-t) pending=uuid.uuid4().hex except Failure: with lock: errors.append(time.monotonic()) time.sleep(.02) with concurrent.futures.ThreadPoolExecutor(4) as pool: workers=[pool.submit(worker,n) for n in range(4)] time.sleep(2) action_start=time.monotonic() try: details=action() action_end=time.monotonic() time.sleep(5) finally: stop.set() for w in workers: w.result() elapsed=time.monotonic()-started data=history(c,b,chat) ids={m['id'] for m in data} assert all(m['id'] in ids for m in confirmed) assert len({m['text'] for m in data})==len(data), 'duplicate message after retry' after=[t for t in times if t>=action_end] assert after gaps=[y-x for x,y in zip(sorted(times),sorted(times)[1:])] assert max(gaps,default=0)<10 return {'duration_seconds':round(elapsed,3),'successful_write_read_pairs':len(times), 'successful_pairs_rps':round(len(times)/elapsed,2),'errors':len(errors), 'first_success_after_action_seconds':round(min(after)-action_end,3), 'max_success_gap_seconds':round(max(gaps,default=0),3), 'action_seconds':round(action_end-action_start,3),'containers':details, 'confirmed_messages_preserved':True} original=command('docker','compose','ps','-q','api').split() for container in original: hostname=json.loads(command('docker','inspect',container))[0]['Config']['Hostname'] def stop_one(): subprocess.run(['scripts/stop-one.sh'],check=True,env=dict(os.environ,TARGET_INSTANCE_ID=hostname),stdout=subprocess.DEVNULL) state=json.loads(command('docker','inspect',container))[0]['State'] assert not state['Running'] return {'stopped':container,'hostname':hostname,'running':state['Running']} try: result['failures'].append(workload(stop_one)) finally: command('docker','start',container) ready() result['rolling']=workload(lambda:json.loads(command(sys.executable,'scripts/rolling.py'))) Path('evidence/grade4.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n') print(json.dumps(result,ensure_ascii=False,indent=2))