156 lines
8.6 KiB
Python
Executable File
156 lines
8.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Explicit lab3 acceptance: includes persistence, process failures and replacements."""
|
|
import argparse,datetime,json,os,subprocess,sys,time,uuid
|
|
from pathlib import Path
|
|
from metrics import ROOT,command,containers
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,Failure,check
|
|
from resilience import history
|
|
|
|
RUN = {}
|
|
|
|
def save(name,value):
|
|
target=ROOT/'evidence'/name
|
|
temporary=target.with_suffix(target.suffix+'.tmp')
|
|
temporary.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
|
temporary.replace(target)
|
|
|
|
def stage(name):
|
|
RUN['stage']=name
|
|
RUN['updated_utc']=datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
save('acceptance.json',RUN)
|
|
|
|
def check_clocks():
|
|
rows=[]
|
|
for ident,name,service in containers():
|
|
if service!='api':continue
|
|
before=time.time()
|
|
remote=int(command(['docker','exec',ident,'date','+%s']))
|
|
after=time.time()
|
|
rows.append({'container':name,'host_epoch_midpoint':(before+after)/2,'api_epoch':remote,'offset_seconds':remote-(before+after)/2})
|
|
check(len(rows)>=2,'acceptance requires at least two running API replicas')
|
|
valid=all(abs(r['offset_seconds'])<=5 for r in rows)
|
|
save('clock-check.json',{'checked_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'maximum_offset_seconds':5,'status':'PASS' if valid else 'failed','api':rows})
|
|
check(valid,'host/API clocks differ by more than 5 seconds; synchronize Docker VM clock before acceptance')
|
|
def state():
|
|
rows=[]
|
|
for ident,name,service in containers():
|
|
r=json.loads(command(['docker','inspect',ident]))[0]
|
|
item={'id':r['Id'],'name':name,'service':service,'image_id':r['Image'],'started_at':r['State']['StartedAt'],'pid':r['State']['Pid'],'cpus':r['HostConfig']['NanoCpus']/1e9,'memory_bytes':r['HostConfig']['Memory'],'published_ports':r['HostConfig']['PortBindings']}
|
|
if service=='api':item['instance_id']=json.loads(command(['docker','exec',ident,'wget','-qO-','http://127.0.0.1:8080/internal/metrics']))['instance_id']
|
|
rows.append(item)
|
|
return rows
|
|
|
|
def restore():
|
|
subprocess.run(['./scripts/compose.sh','up','-d','--wait','--wait-timeout','90','--no-recreate','api','lb'],cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
time.sleep(4)
|
|
|
|
def logrun(filename,args):
|
|
stage(filename)
|
|
print('Running '+filename,flush=True)
|
|
with (ROOT/'evidence'/filename).open('w') as f:
|
|
f.write('$ '+' '.join(args)+'\n');f.flush()
|
|
p=subprocess.run(args,cwd=ROOT,stdout=f,stderr=subprocess.STDOUT,timeout=240)
|
|
if p.returncode:raise RuntimeError(filename+' failed; inspect evidence')
|
|
|
|
def each_failure(base):
|
|
results=[]
|
|
# Each target is chosen by real container identity, independently of LB round-robin phase.
|
|
for name in [r['name'] for r in state() if r['service']=='api']:
|
|
restore();before=state();target=next(r for r in before if r['name']==name)
|
|
c=Client(base,timeout=2);alice,bob,eve,chat=c.bootstrap();original=c.message(alice,chat,'confirmed-before-stop')['json']
|
|
env=dict(os.environ,TARGET_INSTANCE_ID=target['instance_id'])
|
|
started=time.monotonic();proc=subprocess.Popen(['./scripts/stop-one.sh'],cwd=ROOT,env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
|
good=[];read_ok=read_errors=write_errors=0;latencies=[];first=None;instances=set();key=uuid.uuid4().hex;text='inflight-'+key
|
|
try:
|
|
while time.monotonic()-started<15:
|
|
t=time.monotonic()
|
|
try:
|
|
resp=c.message(alice,chat,text,key=key);good.append(resp['json']);latencies.append(time.monotonic()-t)
|
|
if proc.poll()==0:
|
|
if first is None:first=time.monotonic()-started
|
|
instances.add(resp['headers']['X-Instance-Id'])
|
|
key=uuid.uuid4().hex;text='inflight-'+key
|
|
except Failure:write_errors+=1
|
|
try:c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage');read_ok+=1
|
|
except Failure:read_errors+=1
|
|
time.sleep(.03)
|
|
check(proc.wait(timeout=10)==0,'target stop failed')
|
|
check(first is not None and first<=10,'write recovery exceeded 10 seconds')
|
|
check(target['instance_id'] not in instances,'stopped process still responded')
|
|
stopped=json.loads(command(['docker','inspect',target['id']]))[0]
|
|
check(stopped['State']['Status']=='exited','target container was not left stopped')
|
|
messages=history(c,bob,chat);byid={m['id']:m for m in messages}
|
|
check(byid.get(original['id'])==original,'pre-failure confirmation lost')
|
|
check(all(byid.get(m['id'])==m for m in good),'inflight confirmation lost')
|
|
check(len(messages)==len({m['text'] for m in messages}),'retry produced duplicate text')
|
|
elapsed=time.monotonic()-started
|
|
results.append({'target':target,'before':before,'stopped_status':stopped['State']['Status'],'successful_writes':len(good),'successful_reads':read_ok,'write_errors':write_errors,'read_errors':read_errors,'successful_write_rps':len(good)/elapsed,'first_recovery_seconds':first,'confirmed_messages_verified':len(good)+1,'observed_survivors':sorted(instances)})
|
|
save('failover-each.json',results)
|
|
finally:
|
|
if proc.poll() is None:proc.terminate();proc.wait(timeout=10)
|
|
restore()
|
|
return results
|
|
|
|
def accept(base):
|
|
logrun('smoke.txt',['make','test',f'BASE_URL={base}'])
|
|
logrun('go-tests.txt',['make','test-go',f'BASE_URL={base}'])
|
|
logrun('storage-tests.txt',['make','test-storage'])
|
|
save('containers-before-persistence.json',state())
|
|
logrun('persistence.txt',['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh',f'BASE_URL={base}'])
|
|
save('containers-after-persistence.json',state())
|
|
logrun('failover-public.txt',['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh',f'BASE_URL={base}'])
|
|
stage("each API failure")
|
|
restore();each_failure(base)
|
|
# A stable synthetic workload overlaps the complete replacement action.
|
|
stage("rolling replacement")
|
|
c=Client(base);alice,bob,eve,chat=c.bootstrap();confirmed=c.message(alice,chat,'before-rolling')['json']
|
|
before=state();save('containers-before-rolling.json',before)
|
|
with (ROOT/'evidence/rolling-actions.txt').open('w') as f:
|
|
action=subprocess.Popen(['./scripts/rolling-update.sh'],cwd=ROOT,stdout=f,stderr=subprocess.STDOUT)
|
|
sent=[];errors=0;read_ok=read_errors=0;start=time.monotonic();key=uuid.uuid4().hex
|
|
while action.poll() is None:
|
|
if time.monotonic()-start>180:action.terminate();raise RuntimeError('rolling update exceeded budget')
|
|
try:sent.append(c.message(alice,chat,'rolling-'+key,key=key)['json']);key=uuid.uuid4().hex
|
|
except Failure:errors+=1
|
|
try:c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage');read_ok+=1
|
|
except Failure:read_errors+=1
|
|
time.sleep(.03)
|
|
check(action.returncode==0,'rolling action failed')
|
|
elapsed=time.monotonic()-start;messages=history(c,bob,chat);byid={m['id']:m for m in messages}
|
|
check(all(byid.get(m['id'])==m for m in [confirmed]+sent),'rolling lost confirmed messages')
|
|
check(len(messages)==len({m['text'] for m in messages}),'rolling retry duplicated message')
|
|
after=state();save('containers-after-rolling.json',after)
|
|
old={r['id'] for r in before if r['service']=='api'};new={r['id'] for r in after if r['service']=='api'}
|
|
check(not old.intersection(new) and len(new)==len(old),'not all API processes were replaced')
|
|
save('rolling.json',{'successful_writes':len(sent),'successful_reads':read_ok,'write_errors':errors,'read_errors':read_errors,'elapsed_seconds':elapsed,'successful_write_rps':len(sent)/elapsed,'confirmed_messages_verified':len(sent)+1,'old_api_ids':sorted(old),'new_api_ids':sorted(new)})
|
|
logrun('smoke-after-rolling.txt',['make','test',f'BASE_URL={base}'])
|
|
def main():
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--base-url',default='http://localhost:18080')
|
|
args=parser.parse_args()
|
|
(ROOT/'evidence').mkdir(exist_ok=True)
|
|
RUN.update({'run_id':uuid.uuid4().hex,'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'status':'running','stage':'clock preflight'})
|
|
save('acceptance.json',RUN)
|
|
error=None
|
|
try:
|
|
check_clocks()
|
|
accept(args.base_url)
|
|
except BaseException as exc:
|
|
error=exc
|
|
finally:
|
|
failed_stage=RUN['stage']
|
|
try:
|
|
restore()
|
|
except Exception as cleanup:
|
|
if error is None:error=cleanup;failed_stage='restore replicas'
|
|
else:RUN['cleanup_error']=type(cleanup).__name__+': '+str(cleanup)
|
|
RUN.update({'status':'failed' if error else 'PASS','stage':failed_stage if error else 'complete','completed_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()})
|
|
if error:RUN['error']=type(error).__name__+': '+str(error)
|
|
else:RUN['checks']=['clock synchronization','public smoke','Go API tests','storage atomicity/expiry','persistence including stores','public failover','each API failure','rolling replacement under read/write traffic']
|
|
save('acceptance.json',RUN)
|
|
if error:raise error
|
|
print('PASS: lab3 acceptance completed',flush=True)
|
|
|
|
if __name__=='__main__':main()
|