forked from mtusi2026/mtusi2026lab3
102 lines
4.1 KiB
Python
102 lines
4.1 KiB
Python
"""Grade-4 evidence: shared idempotency, each replica failure, rolling load."""
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
os.chdir(ROOT)
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
sys.path.insert(0,str(ROOT/'scripts'))
|
|
from client import Client, Failure
|
|
from resilience import history
|
|
from rolling import command, ready
|
|
base = os.environ.get('BASE_URL','http://localhost:8080')
|
|
ready()
|
|
c = Client(base)
|
|
a,b,_,chat = c.bootstrap()
|
|
key = uuid.uuid4().hex
|
|
barrier = threading.Barrier(16)
|
|
def duplicate(_):
|
|
barrier.wait()
|
|
return Client(base).message(a,chat,'Одна запись при гонке',key=key)
|
|
with concurrent.futures.ThreadPoolExecutor(16) as pool:
|
|
replies = list(pool.map(duplicate,range(16)))
|
|
assert len({r['json']['id'] for r in replies})==1
|
|
instances = {r['headers']['X-Instance-Id'] for r in replies}
|
|
assert len(instances)==2
|
|
assert len(history(c,b,chat))==1
|
|
# Auth and conflict are still checked on replay.
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=a,body={'text':'different'},headers={'Idempotency-Key':key})
|
|
result={'parallel_idempotency':{'requests':16,'instances':sorted(instances),'messages':1}, 'failures':[]}
|
|
|
|
def workload(action):
|
|
stop = threading.Event()
|
|
started=time.monotonic()
|
|
confirmed=[]
|
|
times=[]
|
|
errors=[]
|
|
latencies=[]
|
|
lock=threading.Lock()
|
|
def worker(n):
|
|
local=Client(base,timeout=2)
|
|
pending=uuid.uuid4().hex
|
|
while not stop.is_set():
|
|
t=time.monotonic()
|
|
try:
|
|
msg=local.message(a,chat,pending,key=pending)['json']
|
|
local.call('GET','/api/v1/users/me',actor=a,schema='User')
|
|
with lock:
|
|
confirmed.append(msg); times.append(time.monotonic()); latencies.append(time.monotonic()-t)
|
|
pending=uuid.uuid4().hex
|
|
except Failure:
|
|
with lock: errors.append(time.monotonic())
|
|
time.sleep(.02)
|
|
with concurrent.futures.ThreadPoolExecutor(4) as pool:
|
|
workers=[pool.submit(worker,n) for n in range(4)]
|
|
time.sleep(2)
|
|
action_start=time.monotonic()
|
|
try:
|
|
details=action()
|
|
action_end=time.monotonic()
|
|
time.sleep(5)
|
|
finally:
|
|
stop.set()
|
|
for w in workers: w.result()
|
|
elapsed=time.monotonic()-started
|
|
data=history(c,b,chat)
|
|
ids={m['id'] for m in data}
|
|
assert all(m['id'] in ids for m in confirmed)
|
|
assert len({m['text'] for m in data})==len(data), 'duplicate message after retry'
|
|
after=[t for t in times if t>=action_end]
|
|
assert after
|
|
gaps=[y-x for x,y in zip(sorted(times),sorted(times)[1:])]
|
|
assert max(gaps,default=0)<10
|
|
return {'duration_seconds':round(elapsed,3),'successful_write_read_pairs':len(times),
|
|
'successful_pairs_rps':round(len(times)/elapsed,2),'errors':len(errors),
|
|
'first_success_after_action_seconds':round(min(after)-action_end,3),
|
|
'max_success_gap_seconds':round(max(gaps,default=0),3),
|
|
'action_seconds':round(action_end-action_start,3),'containers':details,
|
|
'confirmed_messages_preserved':True}
|
|
|
|
original=command('docker','compose','ps','-q','api').split()
|
|
for container in original:
|
|
hostname=json.loads(command('docker','inspect',container))[0]['Config']['Hostname']
|
|
def stop_one():
|
|
subprocess.run(['scripts/stop-one.sh'],check=True,env=dict(os.environ,TARGET_INSTANCE_ID=hostname),stdout=subprocess.DEVNULL)
|
|
state=json.loads(command('docker','inspect',container))[0]['State']
|
|
assert not state['Running']
|
|
return {'stopped':container,'hostname':hostname,'running':state['Running']}
|
|
try:
|
|
result['failures'].append(workload(stop_one))
|
|
finally:
|
|
command('docker','start',container)
|
|
ready()
|
|
result['rolling']=workload(lambda:json.loads(command(sys.executable,'scripts/rolling.py')))
|
|
Path('evidence/grade4.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n')
|
|
print(json.dumps(result,ensure_ascii=False,indent=2))
|