Implement observable messenger, SLO alerts and measured incident scenarios
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
"""Shared bounded Docker helpers for reproducible, scoped failure experiments."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
def docker(*args):
|
||||
return subprocess.check_output(['docker',*args],cwd=ROOT,text=True,encoding='utf-8',stderr=subprocess.PIPE).strip()
|
||||
def replicas(): return docker('compose','ps','-q','api').splitlines()
|
||||
|
||||
def ready_replicas(count=2,timeout=60):
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
ids=replicas()
|
||||
if len(ids)==count:
|
||||
states=[json.loads(docker('inspect','--format','{{json .State}}',cid)) for cid in ids]
|
||||
if all(s['Running'] and s.get('Health',{}).get('Status')=='healthy' for s in states): return
|
||||
time.sleep(.5)
|
||||
raise AssertionError('Actual replica health did not recover within timeout')
|
||||
def instance(cid):
|
||||
return docker('exec',cid,'python','-c',"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live',timeout=2).headers['X-Instance-Id'])")
|
||||
def state():
|
||||
result={}
|
||||
for service in ('api','db','cache','lb','s3','reconciler','broker','worker','dispatcher'):
|
||||
for cid in docker('compose','ps','-q',service).splitlines():
|
||||
result[cid]={'service':service,'state':json.loads(docker('inspect','--format','{{json .State}}',cid))}
|
||||
return result
|
||||
def ready(client,timeout=60):
|
||||
deadline=time.monotonic()+timeout
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
if client.get('/health/ready').status_code==200: return
|
||||
except Exception: pass
|
||||
time.sleep(.2)
|
||||
raise AssertionError('Readiness timeout')
|
||||
Reference in New Issue
Block a user