63 lines
3.1 KiB
Python
63 lines
3.1 KiB
Python
"""Run unchanged public resilience checks and record actual container restarts."""
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import subprocess
|
|
import sys
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
def docker(*args):
|
|
return subprocess.check_output(['docker',*args],cwd=ROOT,text=True).strip()
|
|
|
|
def states():
|
|
ids = docker('compose','ps','-q').splitlines()
|
|
rows = json.loads(docker('inspect',*ids))
|
|
return {row['Config']['Labels']['com.docker.compose.service']:
|
|
dict(id=row['Id'][:12],started_at=row['State']['StartedAt'],image=row['Config']['Image'])
|
|
for row in rows}
|
|
|
|
def main():
|
|
base = sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085'
|
|
suffix = '.cmd' if os.name=='nt' else '.sh'
|
|
if os.name!='nt':
|
|
for name in ('restart','stop-one'):
|
|
(ROOT/'scripts'/(name+suffix)).chmod(0o755)
|
|
before = states()
|
|
command = [sys.executable,'tests/resilience.py','--base-url',base]
|
|
result = subprocess.run(command+['--mode','persistence','--action-script','scripts/restart'+suffix],
|
|
cwd=ROOT,capture_output=True,text=True)
|
|
print(result.stdout,end='')
|
|
(ROOT/'evidence/persistence.txt').write_text(result.stdout+result.stderr,encoding='utf-8')
|
|
after = states()
|
|
services = ('api1','api2','db','s3','broker','worker','dispatcher','lb')
|
|
changed = {name:before[name]['started_at']!=after[name]['started_at'] for name in services}
|
|
(ROOT/'evidence/persistence-runtime.json').write_text(json.dumps(dict(before=before,after=after,restarted=changed),indent=2)+'\n',encoding='utf-8')
|
|
if result.returncode or not all(changed.values()):
|
|
raise RuntimeError('persistence verification failed; see evidence')
|
|
failover_results = []
|
|
try:
|
|
for number in range(2):
|
|
result = subprocess.run(command+['--mode','failover','--action-script','scripts/stop-one'+suffix],
|
|
cwd=ROOT,capture_output=True,text=True)
|
|
print(result.stdout,end='')
|
|
failover_results.append(f'Run {number+1}\n'+result.stdout+result.stderr)
|
|
docker('compose','up','-d','--no-deps','api1','api2')
|
|
if result.returncode:
|
|
raise RuntimeError('failover verification failed; see evidence')
|
|
finally:
|
|
docker('compose','up','-d','--no-deps','api1','api2')
|
|
(ROOT/'evidence/failover.txt').write_text('\n'.join(failover_results),encoding='utf-8')
|
|
runtime = dict(python=platform.python_version(),host=platform.platform(),
|
|
docker=json.loads(docker('version','--format','{{json .}}')),
|
|
engine=json.loads(docker('info','--format','{{json .}}')),
|
|
containers=states())
|
|
# Keep only non-sensitive hardware/version fields from Docker info.
|
|
runtime['engine']={key:runtime['engine'][key] for key in ('NCPU','MemTotal','OSType','Architecture','OperatingSystem')}
|
|
(ROOT/'evidence/runtime.json').write_text(json.dumps(runtime,indent=2)+'\n',encoding='utf-8')
|
|
print('PASS: persistence and two API failover runs; actual process restarts recorded')
|
|
|
|
if __name__=='__main__':
|
|
main()
|