forked from mtusi2026/mtusi2026lab4
121 lines
6.4 KiB
Python
121 lines
6.4 KiB
Python
"""Run the complete lab4 demonstration. Only Compose project mtusi-lab4 is mutated.
|
|
|
|
Requires the built stack. Results contain no session tokens or credential values.
|
|
Use --quick to omit load experiments. Never removes volumes.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, check
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--quick',action='store_true')
|
|
args = parser.parse_args()
|
|
os.chdir(ROOT)
|
|
os.environ['PYTHONUTF8'] = '1'
|
|
env_text = (ROOT/'.env').read_text()
|
|
port = next((line.split('=',1)[1] for line in env_text.splitlines() if line.startswith('LAB4_PORT=')),'8084')
|
|
base = 'http://127.0.0.1:'+port
|
|
compose = ['docker','compose','--env-file','.env','-f','compose.yaml','-f','compose.test.yaml','--profile','tests']
|
|
evidence = ROOT/'evidence'
|
|
evidence.mkdir(exist_ok=True)
|
|
def run(name, command, expected=0):
|
|
print('Running '+name+' ...',flush=True)
|
|
result = subprocess.run(command,capture_output=True,text=True,encoding='utf-8',errors='replace')
|
|
(evidence/(name+'.txt')).write_text(result.stdout+result.stderr,encoding='utf-8')
|
|
if result.returncode != expected:
|
|
print(result.stdout+result.stderr)
|
|
raise RuntimeError(name+' failed, exit='+str(result.returncode))
|
|
return result.stdout.strip()
|
|
def ready():
|
|
deadline = time.monotonic()+90
|
|
while time.monotonic()<deadline:
|
|
try:
|
|
Client(base).call('GET','/health/ready',schema='Health')
|
|
return
|
|
except Exception:
|
|
time.sleep(.5)
|
|
raise RuntimeError('Readiness timeout')
|
|
def snapshot(label):
|
|
names = subprocess.check_output(compose+['ps','-q','postgres','s3','api1','api2','lb','sweeper'],text=True).split()
|
|
result = []
|
|
for name in names:
|
|
row = subprocess.check_output(['docker','inspect','--format',
|
|
'{{.Name}}|{{.State.StartedAt}}|{{.State.Status}}|{{.HostConfig.Memory}}|{{.HostConfig.NanoCpus}}',name],text=True).strip().split('|')
|
|
result.append(dict(zip(('name','started_at','status','memory_limit_bytes','nano_cpus'),row)))
|
|
(evidence/(label+'.json')).write_text(json.dumps(result,indent=2),encoding='utf-8')
|
|
return result
|
|
run('check',[sys.executable,'-X','utf8','scripts/check_contract.py'])
|
|
run('compose-config',compose+['config','--quiet'])
|
|
run('test-services',compose+['up','-d','api1','api2','lb','fault-api','sweeper'])
|
|
ready()
|
|
run('smoke',[sys.executable,'-X','utf8','tests/smoke.py','--base-url',base])
|
|
run('acceptance',compose+['run','--rm','--no-deps','test-runner','python','student-tests/acceptance.py'])
|
|
before = snapshot('before-restart')
|
|
suffix = '.cmd' if os.name=='nt' else '.sh'
|
|
if os.name != 'nt':
|
|
for name in ('restart','stop-one'):
|
|
(ROOT/'scripts'/(name+suffix)).chmod(0o755)
|
|
run('persistence',[sys.executable,'-X','utf8','tests/resilience.py','--mode','persistence',
|
|
'--action-script',str(ROOT/'scripts'/('restart'+suffix)),'--base-url',base])
|
|
after = snapshot('after-restart')
|
|
check(all(next(x for x in after if x['name']==row['name'])['started_at']!=row['started_at'] for row in before),'hook did not restart all services')
|
|
run('failover',[sys.executable,'-X','utf8','tests/resilience.py','--mode','failover',
|
|
'--action-script',str(ROOT/'scripts'/('stop-one'+suffix)),'--base-url',base])
|
|
run('restore-api',compose+['start','api1','api2'])
|
|
ready()
|
|
# Explicitly stop the process which uploaded an original, then use another process.
|
|
c = Client(base,timeout=40)
|
|
a,b,e,chat = c.bootstrap()
|
|
upload_client = Client(base)
|
|
response = upload_client.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=failover.png',201,
|
|
actor=a,raw=(ROOT/'tests/fixtures/sample.png').read_bytes(),headers={'Content-Type':'image/png'},schema='Attachment')
|
|
target = response['headers']['X-Instance-Id']
|
|
run('stop-upload-replica',compose+['stop','-t','2',target])
|
|
ready()
|
|
download = c.call('GET','/api/v1/attachments/'+response['json']['id']+'/content',actor=b)
|
|
check(download['headers']['X-Instance-Id']!=target,'download remained on uploader')
|
|
check(download['bytes']==(ROOT/'tests/fixtures/sample.png').read_bytes(),'cross-replica bytes mismatch')
|
|
(evidence/'upload-failover.json').write_text(json.dumps({'status':'PASS','upload_instance':target,
|
|
'download_instance':download['headers']['X-Instance-Id'],'checksum_matches':True}),encoding='utf-8')
|
|
run('restore-upload-replica',compose+['start',target])
|
|
ready()
|
|
# Dependencies unavailable: both upload and reconciliation must fail closed.
|
|
for dependency in ('s3','postgres'):
|
|
run('stop-'+dependency,compose+['stop','-t','2',dependency])
|
|
try:
|
|
direct = compose+['run','--rm','--no-deps','test-runner','python','student-tests/outage.py',dependency]
|
|
run(dependency+'-outage',direct)
|
|
run(dependency+'-reconcile-abort',compose+['run','--rm','--no-deps','reconciler'],expected=1)
|
|
finally:
|
|
run('restore-'+dependency,compose+['start',dependency])
|
|
ready()
|
|
run('reconciliation',compose+['run','--rm','--no-deps','test-runner','python','student-tests/reconciliation.py'])
|
|
run('bootstrap-repeat',compose+['run','--rm','--no-deps','bootstrap'])
|
|
if not args.quick:
|
|
run('upload-load',compose+['run','--rm','--no-deps','test-runner','python','student-tests/upload_load.py'])
|
|
run('message-load',[sys.executable,'-X','utf8','tests/load.py','--base-url',base,
|
|
'--duration','15','--concurrency','4','--output','evidence/message-load.json'])
|
|
snapshot('final-containers')
|
|
run('final-smoke',[sys.executable,'-X','utf8','tests/smoke.py','--base-url',base])
|
|
run('stop-fault-api',compose+['stop','fault-api'])
|
|
(evidence/'verification.json').write_text(json.dumps({'status':'PASS',
|
|
'completed_at':dt.datetime.now(dt.timezone.utc).isoformat(), 'load_experiments':not args.quick,
|
|
'project':'mtusi-lab4','base_url':base},indent=2),encoding='utf-8')
|
|
print('PASS: full lab4 verification; results in evidence/.')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|