60 lines
2.2 KiB
Python
60 lines
2.2 KiB
Python
"""A live process with lost DB network must be excluded by the LB."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client
|
|
|
|
|
|
def cmd(*args):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
|
|
|
|
|
def main():
|
|
c = Client("http://127.0.0.1:8083")
|
|
actor = c.user("readiness")
|
|
target = cmd("docker", "compose", "ps", "-q", "api").split()[0]
|
|
raw = json.loads(cmd("docker", "inspect", target))[0]
|
|
backend = next(n for n in raw["NetworkSettings"]["Networks"] if n.endswith("_backend"))
|
|
probe = """import json,urllib.request,urllib.error
|
|
try: r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=8)
|
|
except urllib.error.HTTPError as e: r=e
|
|
print(json.dumps({'status':r.code,'alias':r.headers['X-Instance-Id']}))"""
|
|
initial = json.loads(cmd("docker", "exec", target, "python", "-c", probe))
|
|
assert initial["status"] == 200
|
|
started = time.monotonic()
|
|
cmd("docker", "network", "disconnect", backend, target)
|
|
try:
|
|
failed = json.loads(cmd("docker", "exec", target, "python", "-c", probe))
|
|
assert failed["status"] == 503
|
|
time.sleep(4)
|
|
observations = []
|
|
for _ in range(30):
|
|
response = c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
alias = response["headers"]["X-Instance-Id"]
|
|
assert alias != initial["alias"]
|
|
observations.append(alias)
|
|
result = {
|
|
"target": target,
|
|
"failed_instance": initial["alias"],
|
|
"direct_readiness": 503,
|
|
"seconds_until_verification": time.monotonic() - started,
|
|
"lb_successful_requests": len(observations),
|
|
"lb_instances": sorted(set(observations)),
|
|
}
|
|
finally:
|
|
cmd("docker", "network", "connect", "--alias", "api", backend, target)
|
|
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
|
result["recovered_two_instances"] = True
|
|
(ROOT / "evidence/readiness-isolation.json").write_text(json.dumps(result, indent=2) + "\n")
|
|
print(json.dumps(result, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|