175 lines
7.9 KiB
Python
175 lines
7.9 KiB
Python
"""Reproduce evidence on a LOCAL synthetic Compose stand, Python 3.9+ and Docker.
|
|
|
|
No secrets in summaries. Fails immediately on a broken check; restores stopped services.
|
|
"""
|
|
import concurrent.futures
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
OUT = ROOT/'evidence'
|
|
OUT.mkdir(exist_ok=True)
|
|
os.chdir(ROOT)
|
|
os.environ['PYTHONUTF8'] = '1'
|
|
|
|
|
|
def command(args, file=None):
|
|
result = subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
|
|
text=True,encoding='utf-8',errors='replace')
|
|
if file:
|
|
(OUT/file).write_text(result.stdout,encoding='utf-8')
|
|
if result.returncode:
|
|
print(result.stdout+result.stderr)
|
|
raise RuntimeError('Command failed: '+' '.join(args[:5]))
|
|
return result.stdout
|
|
|
|
|
|
def ready():
|
|
for _ in range(90):
|
|
try:
|
|
with urllib.request.urlopen(os.environ['BASE_URL']+'/health/ready',timeout=3) as r:
|
|
if r.status==200:
|
|
return
|
|
except (OSError,urllib.error.URLError):
|
|
pass
|
|
time.sleep(1)
|
|
raise RuntimeError('Readiness deadline')
|
|
|
|
|
|
def snapshot(name):
|
|
# No environment/inspect Config: Docker environments can contain secrets.
|
|
data = command(['docker','compose','ps','--all','--format','json'])
|
|
# Compose emits JSON Lines; evidence files must be valid standalone JSON.
|
|
try:
|
|
parsed = json.loads(data)
|
|
rows = parsed if isinstance(parsed,list) else [parsed]
|
|
except json.JSONDecodeError:
|
|
rows = [json.loads(line) for line in data.splitlines() if line.strip()]
|
|
(OUT/name).write_text(json.dumps(rows,indent=2),encoding='utf-8')
|
|
starts = command(['docker','inspect','--format',
|
|
'{{.Name}} {{.State.StartedAt}}',
|
|
'chernenko-lab4-postgres-1','chernenko-lab4-s3-1',
|
|
'chernenko-lab4-api1-1','chernenko-lab4-api2-1','chernenko-lab4-lb-1'])
|
|
(OUT/(Path(name).stem+'-started.txt')).write_text(starts,encoding='utf-8')
|
|
|
|
|
|
def execute_inside(script,filename):
|
|
return command(['docker','compose','exec','-T','api1','python',script],filename)
|
|
|
|
|
|
def main():
|
|
command(['docker','compose','config','--quiet'],'compose-config.txt')
|
|
(OUT/'compose-config.txt').write_text('PASS: docker compose config --quiet (exit 0)\n',encoding='utf-8')
|
|
address = command(['docker','compose','port','lb','8080']).strip()
|
|
os.environ['BASE_URL'] = 'http://'+address
|
|
command(['docker','compose','exec','-T','api1','make','check','validate','PYTHON=python'],'check-validate.txt')
|
|
ready()
|
|
command(['docker','compose','exec','-T','api1','make','test','PYTHON=python','BASE_URL=http://lb:8080'],'smoke.txt')
|
|
execute_inside('student-tests/acceptance.py','acceptance.json')
|
|
snapshot('before-restart.json')
|
|
hook = 'scripts/restart.cmd' if os.name=='nt' else 'scripts/restart.sh'
|
|
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','persistence',
|
|
'--action-script',hook],'persistence.txt')
|
|
snapshot('after-restart.json')
|
|
# Both failure directions deterministically, including bytes and existing session.
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client, check
|
|
c = Client(timeout=20)
|
|
a,b,e,ch = c.bootstrap()
|
|
att,raw = c.attachment(a,ch)
|
|
c.message(a,ch,'survives',attachments=[att['id']])
|
|
outages = []
|
|
for target in ('api1','api2'):
|
|
command(['docker','compose','stop',target])
|
|
try:
|
|
time.sleep(3)
|
|
result = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)
|
|
check(result['bytes']==raw,'failover changed bytes')
|
|
check(result['headers']['X-Instance-Id']!=target,'stopped replica served response')
|
|
c.message(a,ch,'after '+target)
|
|
outages.append(dict(stopped=target,served_by=result['headers']['X-Instance-Id'],checksum_equal=True))
|
|
finally:
|
|
command(['docker','compose','start',target])
|
|
time.sleep(4)
|
|
ready()
|
|
hook = 'scripts/stop-one.cmd' if os.name=='nt' else 'scripts/stop-one.sh'
|
|
try:
|
|
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','failover',
|
|
'--action-script',hook],'failover.txt')
|
|
snapshot('after-failover.json')
|
|
finally:
|
|
command(['docker','compose','start','api1','api2'])
|
|
time.sleep(4)
|
|
ready()
|
|
# Internal API avoids LB-generated 503 masking application behavior. Sessions
|
|
# are passed only over stdin, never saved in evidence or process arguments.
|
|
for dependency in ('s3','postgres'):
|
|
try:
|
|
command(['docker','compose','stop',dependency])
|
|
result = subprocess.run(['docker','compose','exec','-T','api2','python',
|
|
'student-tests/storage_outage.py',dependency],input=json.dumps(
|
|
{'actor':{'id':a['id'],'token':a['token']},'chat':ch,'attachment':att}),text=True,encoding='utf-8',
|
|
stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
|
(OUT/(dependency+'-outage.json')).write_text(result.stdout,encoding='utf-8')
|
|
if result.returncode:
|
|
raise RuntimeError(dependency+' outage assertions failed: '+result.stdout)
|
|
finally:
|
|
command(['docker','compose','start',dependency])
|
|
time.sleep(4)
|
|
ready()
|
|
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==raw,
|
|
'storage restart changed original')
|
|
(OUT/'replica-directions.json').write_text(json.dumps(outages,indent=2),encoding='utf-8')
|
|
execute_inside('student-tests/research.py','research.json')
|
|
samples = []
|
|
done = threading.Event()
|
|
def sample():
|
|
while not done.is_set():
|
|
raw = command(['docker','stats','--no-stream','--format','{{json .}}',
|
|
'chernenko-lab4-api1-1','chernenko-lab4-api2-1'])
|
|
samples.append({'elapsed':round(time.monotonic()-start,3),
|
|
'replicas':[json.loads(x) for x in raw.splitlines() if x.strip()]})
|
|
done.wait(0.2)
|
|
start = time.monotonic()
|
|
thread = threading.Thread(target=sample)
|
|
thread.start()
|
|
probes = {service:subprocess.Popen(['docker','compose','exec','-T',service,'python',
|
|
'student-tests/memory_probe.py'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,text=True,encoding='utf-8') for service in ('api1','api2')}
|
|
try:
|
|
# Separate generator container: its image generation RAM is not charged
|
|
# to either measured API replica.
|
|
command(['docker','compose','run','--rm','--no-deps','migrate','python',
|
|
'student-tests/upload_load.py'],'uploads.json')
|
|
finally:
|
|
done.set()
|
|
thread.join()
|
|
(OUT/'memory.json').write_text(json.dumps(samples,indent=2),encoding='utf-8')
|
|
peaks = {}
|
|
for service,process in probes.items():
|
|
output,error = process.communicate(input='done',timeout=10)
|
|
if process.returncode:
|
|
raise RuntimeError('Memory probe failed: '+error)
|
|
peaks[service] = json.loads(output)
|
|
(OUT/'memory-peaks.json').write_text(json.dumps(peaks,indent=2),encoding='utf-8')
|
|
command([sys.executable,'-X','utf8','tests/load.py','--duration','30','--concurrency','4',
|
|
'--output','evidence/messages-load.json'],'messages-load.txt')
|
|
command(['docker','info','--format','CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}'],
|
|
'hardware.txt')
|
|
command(['docker','compose','images'],'images.txt')
|
|
command(['docker','compose','exec','-T','api1','python','-m','pip','freeze'],'python-packages.txt')
|
|
command(['docker','compose','run','--rm','bootstrap'],'bootstrap.txt')
|
|
snapshot('final-state.json')
|
|
print('PASS: all lab4 scenarios. Sanitized evidence in evidence/')
|
|
|
|
|
|
if __name__=='__main__':
|
|
main()
|