Files
mtusi2026lab6/scripts/test-db-incident.py
T

176 lines
3.9 KiB
Python

import base64
import json
import subprocess
import time
import urllib.error
import urllib.request
import uuid
BASE_URL = "http://127.0.0.1:8080"
def docker(*args):
subprocess.run(
["docker", "compose", *args],
check=True,
)
def request(method, path, body=None, headers=None):
request_headers = dict(headers or {})
if isinstance(body, dict):
body = json.dumps(body).encode("utf-8")
request_headers["Content-Type"] = "application/json"
req = urllib.request.Request(
BASE_URL + path,
data=body,
headers=request_headers,
method=method,
)
started = time.monotonic()
try:
with urllib.request.urlopen(req, timeout=5) as response:
response.read()
return response.status, time.monotonic() - started
except urllib.error.HTTPError as exc:
exc.read()
return exc.code, time.monotonic() - started
except (urllib.error.URLError, TimeoutError) as exc:
return type(exc).__name__, time.monotonic() - started
def login_header(username, password):
encoded = base64.b64encode(
f"{username}:{password}".encode("utf-8")
).decode("ascii")
return {
"Authorization": f"Basic {encoded}"
}
def wait_for_recovery(headers):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
status, duration = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
if status == 201:
print(
f"Recovery: HTTP 201, {duration:.3f}s",
flush=True,
)
return
time.sleep(2)
raise RuntimeError(
"Authentication did not recover after DB restart"
)
def main():
suffix = uuid.uuid4().hex[:12]
username = "lab6_db_" + suffix
password = "IncidentTest_" + suffix
status, duration = request(
"POST",
"/api/v1/users",
body={
"username": username,
"display_name": "DB incident test",
"password": password,
},
)
if status != 201:
raise RuntimeError(
f"Cannot create test user: HTTP {status}"
)
headers = login_header(username, password)
status, _ = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
if status != 201:
raise RuntimeError(
f"Baseline login failed: HTTP {status}"
)
print("Baseline: authentication HTTP 201", flush=True)
observed_failures = 0
print("INCIDENT START: stopping PostgreSQL", flush=True)
try:
docker("stop", "db")
for attempt in range(1, 6):
status, duration = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
print(
f"Probe {attempt}: status={status}, "
f"duration={duration:.3f}s",
flush=True,
)
if status != 201:
observed_failures += 1
time.sleep(2)
finally:
print(
"INCIDENT END: restoring PostgreSQL",
flush=True,
)
docker("up", "-d", "--wait", "db")
wait_for_recovery(headers)
print(
f"Failed requests during incident: "
f"{observed_failures}/5",
flush=True,
)
if observed_failures == 0:
raise RuntimeError(
"The database incident did not affect requests"
)
print(
"PASS: database failure observed and recovered",
flush=True,
)
if __name__ == "__main__":
main()