Files
mtusi2026lab4/scripts/run-all.py

175 lines
7.9 KiB
Python

"""Reproduce evidence on a LOCAL synthetic Compose stand, Python 3.9+ and Docker.
No secrets in summaries. Fails immediately on a broken check; restores stopped services.
"""
import concurrent.futures
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT/'evidence'
OUT.mkdir(exist_ok=True)
os.chdir(ROOT)
os.environ['PYTHONUTF8'] = '1'
def command(args, file=None):
result = subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
text=True,encoding='utf-8',errors='replace')
if file:
(OUT/file).write_text(result.stdout,encoding='utf-8')
if result.returncode:
print(result.stdout+result.stderr)
raise RuntimeError('Command failed: '+' '.join(args[:5]))
return result.stdout
def ready():
for _ in range(90):
try:
with urllib.request.urlopen(os.environ['BASE_URL']+'/health/ready',timeout=3) as r:
if r.status==200:
return
except (OSError,urllib.error.URLError):
pass
time.sleep(1)
raise RuntimeError('Readiness deadline')
def snapshot(name):
# No environment/inspect Config: Docker environments can contain secrets.
data = command(['docker','compose','ps','--all','--format','json'])
# Compose emits JSON Lines; evidence files must be valid standalone JSON.
try:
parsed = json.loads(data)
rows = parsed if isinstance(parsed,list) else [parsed]
except json.JSONDecodeError:
rows = [json.loads(line) for line in data.splitlines() if line.strip()]
(OUT/name).write_text(json.dumps(rows,indent=2),encoding='utf-8')
starts = command(['docker','inspect','--format',
'{{.Name}} {{.State.StartedAt}}',
'chernenko-lab4-postgres-1','chernenko-lab4-s3-1',
'chernenko-lab4-api1-1','chernenko-lab4-api2-1','chernenko-lab4-lb-1'])
(OUT/(Path(name).stem+'-started.txt')).write_text(starts,encoding='utf-8')
def execute_inside(script,filename):
return command(['docker','compose','exec','-T','api1','python',script],filename)
def main():
command(['docker','compose','config','--quiet'],'compose-config.txt')
(OUT/'compose-config.txt').write_text('PASS: docker compose config --quiet (exit 0)\n',encoding='utf-8')
address = command(['docker','compose','port','lb','8080']).strip()
os.environ['BASE_URL'] = 'http://'+address
command(['docker','compose','exec','-T','api1','make','check','validate','PYTHON=python'],'check-validate.txt')
ready()
command(['docker','compose','exec','-T','api1','make','test','PYTHON=python','BASE_URL=http://lb:8080'],'smoke.txt')
execute_inside('student-tests/acceptance.py','acceptance.json')
snapshot('before-restart.json')
hook = 'scripts/restart.cmd' if os.name=='nt' else 'scripts/restart.sh'
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','persistence',
'--action-script',hook],'persistence.txt')
snapshot('after-restart.json')
# Both failure directions deterministically, including bytes and existing session.
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, check
c = Client(timeout=20)
a,b,e,ch = c.bootstrap()
att,raw = c.attachment(a,ch)
c.message(a,ch,'survives',attachments=[att['id']])
outages = []
for target in ('api1','api2'):
command(['docker','compose','stop',target])
try:
time.sleep(3)
result = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)
check(result['bytes']==raw,'failover changed bytes')
check(result['headers']['X-Instance-Id']!=target,'stopped replica served response')
c.message(a,ch,'after '+target)
outages.append(dict(stopped=target,served_by=result['headers']['X-Instance-Id'],checksum_equal=True))
finally:
command(['docker','compose','start',target])
time.sleep(4)
ready()
hook = 'scripts/stop-one.cmd' if os.name=='nt' else 'scripts/stop-one.sh'
try:
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','failover',
'--action-script',hook],'failover.txt')
snapshot('after-failover.json')
finally:
command(['docker','compose','start','api1','api2'])
time.sleep(4)
ready()
# Internal API avoids LB-generated 503 masking application behavior. Sessions
# are passed only over stdin, never saved in evidence or process arguments.
for dependency in ('s3','postgres'):
try:
command(['docker','compose','stop',dependency])
result = subprocess.run(['docker','compose','exec','-T','api2','python',
'student-tests/storage_outage.py',dependency],input=json.dumps(
{'actor':{'id':a['id'],'token':a['token']},'chat':ch,'attachment':att}),text=True,encoding='utf-8',
stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
(OUT/(dependency+'-outage.json')).write_text(result.stdout,encoding='utf-8')
if result.returncode:
raise RuntimeError(dependency+' outage assertions failed: '+result.stdout)
finally:
command(['docker','compose','start',dependency])
time.sleep(4)
ready()
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==raw,
'storage restart changed original')
(OUT/'replica-directions.json').write_text(json.dumps(outages,indent=2),encoding='utf-8')
execute_inside('student-tests/research.py','research.json')
samples = []
done = threading.Event()
def sample():
while not done.is_set():
raw = command(['docker','stats','--no-stream','--format','{{json .}}',
'chernenko-lab4-api1-1','chernenko-lab4-api2-1'])
samples.append({'elapsed':round(time.monotonic()-start,3),
'replicas':[json.loads(x) for x in raw.splitlines() if x.strip()]})
done.wait(0.2)
start = time.monotonic()
thread = threading.Thread(target=sample)
thread.start()
probes = {service:subprocess.Popen(['docker','compose','exec','-T',service,'python',
'student-tests/memory_probe.py'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,
stderr=subprocess.PIPE,text=True,encoding='utf-8') for service in ('api1','api2')}
try:
# Separate generator container: its image generation RAM is not charged
# to either measured API replica.
command(['docker','compose','run','--rm','--no-deps','migrate','python',
'student-tests/upload_load.py'],'uploads.json')
finally:
done.set()
thread.join()
(OUT/'memory.json').write_text(json.dumps(samples,indent=2),encoding='utf-8')
peaks = {}
for service,process in probes.items():
output,error = process.communicate(input='done',timeout=10)
if process.returncode:
raise RuntimeError('Memory probe failed: '+error)
peaks[service] = json.loads(output)
(OUT/'memory-peaks.json').write_text(json.dumps(peaks,indent=2),encoding='utf-8')
command([sys.executable,'-X','utf8','tests/load.py','--duration','30','--concurrency','4',
'--output','evidence/messages-load.json'],'messages-load.txt')
command(['docker','info','--format','CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}'],
'hardware.txt')
command(['docker','compose','images'],'images.txt')
command(['docker','compose','exec','-T','api1','python','-m','pip','freeze'],'python-packages.txt')
command(['docker','compose','run','--rm','bootstrap'],'bootstrap.txt')
snapshot('final-state.json')
print('PASS: all lab4 scenarios. Sanitized evidence in evidence/')
if __name__=='__main__':
main()