#!/usr/bin/env python3 """Explicit lab3 acceptance: includes persistence, process failures and replacements.""" import argparse,datetime,json,os,subprocess,sys,time,uuid from pathlib import Path from metrics import ROOT,command,containers sys.path.insert(0,str(ROOT/'tests')) from client import Client,Failure,check from resilience import history RUN = {} def save(name,value): target=ROOT/'evidence'/name temporary=target.with_suffix(target.suffix+'.tmp') temporary.write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') temporary.replace(target) def stage(name): RUN['stage']=name RUN['updated_utc']=datetime.datetime.now(datetime.timezone.utc).isoformat() save('acceptance.json',RUN) def check_clocks(): rows=[] for ident,name,service in containers(): if service!='api':continue before=time.time() remote=int(command(['docker','exec',ident,'date','+%s'])) after=time.time() rows.append({'container':name,'host_epoch_midpoint':(before+after)/2,'api_epoch':remote,'offset_seconds':remote-(before+after)/2}) check(len(rows)>=2,'acceptance requires at least two running API replicas') valid=all(abs(r['offset_seconds'])<=5 for r in rows) save('clock-check.json',{'checked_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'maximum_offset_seconds':5,'status':'PASS' if valid else 'failed','api':rows}) check(valid,'host/API clocks differ by more than 5 seconds; synchronize Docker VM clock before acceptance') def state(): rows=[] for ident,name,service in containers(): r=json.loads(command(['docker','inspect',ident]))[0] item={'id':r['Id'],'name':name,'service':service,'image_id':r['Image'],'started_at':r['State']['StartedAt'],'pid':r['State']['Pid'],'cpus':r['HostConfig']['NanoCpus']/1e9,'memory_bytes':r['HostConfig']['Memory'],'published_ports':r['HostConfig']['PortBindings']} if service=='api':item['instance_id']=json.loads(command(['docker','exec',ident,'wget','-qO-','http://127.0.0.1:8080/internal/metrics']))['instance_id'] rows.append(item) return rows def restore(): subprocess.run(['./scripts/compose.sh','up','-d','--wait','--wait-timeout','90','--no-recreate','api','lb'],cwd=ROOT,check=True,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) time.sleep(4) def logrun(filename,args): stage(filename) print('Running '+filename,flush=True) with (ROOT/'evidence'/filename).open('w') as f: f.write('$ '+' '.join(args)+'\n');f.flush() p=subprocess.run(args,cwd=ROOT,stdout=f,stderr=subprocess.STDOUT,timeout=240) if p.returncode:raise RuntimeError(filename+' failed; inspect evidence') def each_failure(base): results=[] # Each target is chosen by real container identity, independently of LB round-robin phase. for name in [r['name'] for r in state() if r['service']=='api']: restore();before=state();target=next(r for r in before if r['name']==name) c=Client(base,timeout=2);alice,bob,eve,chat=c.bootstrap();original=c.message(alice,chat,'confirmed-before-stop')['json'] env=dict(os.environ,TARGET_INSTANCE_ID=target['instance_id']) started=time.monotonic();proc=subprocess.Popen(['./scripts/stop-one.sh'],cwd=ROOT,env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL) good=[];read_ok=read_errors=write_errors=0;latencies=[];first=None;instances=set();key=uuid.uuid4().hex;text='inflight-'+key try: while time.monotonic()-started<15: t=time.monotonic() try: resp=c.message(alice,chat,text,key=key);good.append(resp['json']);latencies.append(time.monotonic()-t) if proc.poll()==0: if first is None:first=time.monotonic()-started instances.add(resp['headers']['X-Instance-Id']) key=uuid.uuid4().hex;text='inflight-'+key except Failure:write_errors+=1 try:c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage');read_ok+=1 except Failure:read_errors+=1 time.sleep(.03) check(proc.wait(timeout=10)==0,'target stop failed') check(first is not None and first<=10,'write recovery exceeded 10 seconds') check(target['instance_id'] not in instances,'stopped process still responded') stopped=json.loads(command(['docker','inspect',target['id']]))[0] check(stopped['State']['Status']=='exited','target container was not left stopped') messages=history(c,bob,chat);byid={m['id']:m for m in messages} check(byid.get(original['id'])==original,'pre-failure confirmation lost') check(all(byid.get(m['id'])==m for m in good),'inflight confirmation lost') check(len(messages)==len({m['text'] for m in messages}),'retry produced duplicate text') elapsed=time.monotonic()-started results.append({'target':target,'before':before,'stopped_status':stopped['State']['Status'],'successful_writes':len(good),'successful_reads':read_ok,'write_errors':write_errors,'read_errors':read_errors,'successful_write_rps':len(good)/elapsed,'first_recovery_seconds':first,'confirmed_messages_verified':len(good)+1,'observed_survivors':sorted(instances)}) save('failover-each.json',results) finally: if proc.poll() is None:proc.terminate();proc.wait(timeout=10) restore() return results def accept(base): logrun('smoke.txt',['make','test',f'BASE_URL={base}']) logrun('go-tests.txt',['make','test-go',f'BASE_URL={base}']) logrun('storage-tests.txt',['make','test-storage']) save('containers-before-persistence.json',state()) logrun('persistence.txt',['make','test-persistence','ACTION_SCRIPT=scripts/restart.sh',f'BASE_URL={base}']) save('containers-after-persistence.json',state()) logrun('failover-public.txt',['make','test-failover','ACTION_SCRIPT=scripts/stop-one.sh',f'BASE_URL={base}']) stage("each API failure") restore();each_failure(base) # A stable synthetic workload overlaps the complete replacement action. stage("rolling replacement") c=Client(base);alice,bob,eve,chat=c.bootstrap();confirmed=c.message(alice,chat,'before-rolling')['json'] before=state();save('containers-before-rolling.json',before) with (ROOT/'evidence/rolling-actions.txt').open('w') as f: action=subprocess.Popen(['./scripts/rolling-update.sh'],cwd=ROOT,stdout=f,stderr=subprocess.STDOUT) sent=[];errors=0;read_ok=read_errors=0;start=time.monotonic();key=uuid.uuid4().hex while action.poll() is None: if time.monotonic()-start>180:action.terminate();raise RuntimeError('rolling update exceeded budget') try:sent.append(c.message(alice,chat,'rolling-'+key,key=key)['json']);key=uuid.uuid4().hex except Failure:errors+=1 try:c.call('GET','/api/v1/chats/'+chat['id']+'/messages',actor=bob,schema='MessagePage');read_ok+=1 except Failure:read_errors+=1 time.sleep(.03) check(action.returncode==0,'rolling action failed') elapsed=time.monotonic()-start;messages=history(c,bob,chat);byid={m['id']:m for m in messages} check(all(byid.get(m['id'])==m for m in [confirmed]+sent),'rolling lost confirmed messages') check(len(messages)==len({m['text'] for m in messages}),'rolling retry duplicated message') after=state();save('containers-after-rolling.json',after) old={r['id'] for r in before if r['service']=='api'};new={r['id'] for r in after if r['service']=='api'} check(not old.intersection(new) and len(new)==len(old),'not all API processes were replaced') save('rolling.json',{'successful_writes':len(sent),'successful_reads':read_ok,'write_errors':errors,'read_errors':read_errors,'elapsed_seconds':elapsed,'successful_write_rps':len(sent)/elapsed,'confirmed_messages_verified':len(sent)+1,'old_api_ids':sorted(old),'new_api_ids':sorted(new)}) logrun('smoke-after-rolling.txt',['make','test',f'BASE_URL={base}']) def main(): parser=argparse.ArgumentParser() parser.add_argument('--base-url',default='http://localhost:18080') args=parser.parse_args() (ROOT/'evidence').mkdir(exist_ok=True) RUN.update({'run_id':uuid.uuid4().hex,'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'status':'running','stage':'clock preflight'}) save('acceptance.json',RUN) error=None try: check_clocks() accept(args.base_url) except BaseException as exc: error=exc finally: failed_stage=RUN['stage'] try: restore() except Exception as cleanup: if error is None:error=cleanup;failed_stage='restore replicas' else:RUN['cleanup_error']=type(cleanup).__name__+': '+str(cleanup) RUN.update({'status':'failed' if error else 'PASS','stage':failed_stage if error else 'complete','completed_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()}) if error:RUN['error']=type(error).__name__+': '+str(error) else:RUN['checks']=['clock synchronization','public smoke','Go API tests','storage atomicity/expiry','persistence including stores','public failover','each API failure','rolling replacement under read/write traffic'] save('acceptance.json',RUN) if error:raise error print('PASS: lab3 acceptance completed',flush=True) if __name__=='__main__':main()