Files
mtusi2026lab6/scripts/lab6_incidents.py
T

223 lines
7.5 KiB
Python

#!/usr/bin/env python3
"""Lab6 failure scenarios. Prints statuses, ids and metric values. Never tokens."""
import base64
import json
import os
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
PROM = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
def compose(*args: str) -> None:
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
if os.environ.get("COMPOSE_ALTPORTS") == "1":
command.extend(["-f", "compose.altports.yaml"])
subprocess.run([*command, *args], cwd=ROOT, check=True)
class Api:
def __init__(self) -> None:
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None, timeout=15):
hdrs = dict(headers or {})
data = raw
if body is not None:
data = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if content_type:
hdrs["Content-Type"] = content_type
if token:
hdrs["Authorization"] = "Bearer " + token
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
try:
response = self.opener.open(request, timeout=timeout)
except urllib.error.HTTPError as error:
response = error
with response:
payload = response.read()
return response.status, payload
def expect(status, got, label):
if got != status:
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
print("PASS", label, got)
def prom_query(expr: str):
url = PROM + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
with urllib.request.urlopen(url, timeout=10) as response:
body = json.loads(response.read().decode())
if body.get("status") != "success":
raise SystemExit("FAIL prometheus query")
result = body["data"]["result"]
if not result:
return None
return float(result[0]["value"][1])
def alerts():
with urllib.request.urlopen(PROM + "/api/v1/alerts", timeout=10) as response:
body = json.loads(response.read().decode())
names = []
for item in body["data"]["alerts"]:
if item.get("state") == "firing":
names.append(item["labels"].get("alertname", ""))
return sorted(names)
def wait_ready_http():
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
status, _ = Api().call("GET", "/health/ready", timeout=3)
if status == 200:
return
except Exception:
pass
time.sleep(1)
raise SystemExit("FAIL API did not become ready")
def user(api: Api):
username = "l6_" + uuid.uuid4().hex[:10]
password = "T3st-" + uuid.uuid4().hex
status, _ = api.call(
"POST",
"/api/v1/users",
{"username": username, "display_name": "Lab6", "password": password},
)
expect(201, status, "register")
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, payload = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
expect(201, status, "login")
return json.loads(payload)["token"]
def main() -> None:
api = Api()
token = user(api)
status, chat_raw = api.call(
"POST",
"/api/v1/chats",
{"title": "lab6-incidents", "member_ids": []},
token=token,
)
# member_ids must include someone? empty may 400. Create with only self via API rules.
if status != 201:
me_status, me_raw = api.call("GET", "/api/v1/users/me", token=token)
expect(200, me_status, "me")
me = json.loads(me_raw)["id"]
status, chat_raw = api.call(
"POST",
"/api/v1/chats",
{"title": "lab6-incidents", "member_ids": [me]},
token=token,
)
expect(201, status, "chat")
chat_id = json.loads(chat_raw)["id"]
print("chat", chat_id)
print("-- worker stopped: queue age grows, content stays 409 --")
compose("stop", "worker")
content = FIXTURE.read_bytes()
status, attachment_raw = api.call(
"POST",
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
token=token,
raw=content,
content_type="image/png",
)
expect(202, status, "upload while worker stopped")
attachment_id = json.loads(attachment_raw)["id"]
print("attachment", attachment_id)
status, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment_id, token=token)
expect(409, status, "content before ready")
deadline = time.monotonic() + 80
age = None
while time.monotonic() < deadline:
age = prom_query("max(messenger_queue_oldest_age_seconds)")
print("queue_oldest_age_seconds", age, "firing", ",".join(alerts()) or "-")
if age is not None and age >= 15:
break
time.sleep(5)
if age is None or age < 5:
raise SystemExit("FAIL queue age did not grow")
compose("start", "worker")
ready_deadline = time.monotonic() + 90
while time.monotonic() < ready_deadline:
status, body = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
expect(200, status, "poll")
parsed = json.loads(body)
if parsed["status"] == "ready":
print("PASS attachment ready after worker start", attachment_id)
break
if parsed["status"] == "failed":
raise SystemExit("FAIL attachment failed")
time.sleep(0.5)
else:
raise SystemExit("FAIL attachment did not become ready")
time.sleep(12)
print("queue_oldest_age_seconds_after", prom_query("max(messenger_queue_oldest_age_seconds)"))
print("-- postgres stopped: live stays 200, ready becomes 503 --")
compose("stop", "postgres")
time.sleep(2)
status, _ = api.call("GET", "/health/live")
expect(200, status, "live while postgres stopped")
status, _ = api.call("GET", "/health/ready")
expect(503, status, "ready while postgres stopped")
burst_503 = 0
for _ in range(8):
status, _ = api.call(
"POST",
"/api/v1/users",
{
"username": "l6d_" + uuid.uuid4().hex[:8],
"display_name": "DbDown",
"password": "T3st-" + uuid.uuid4().hex,
},
)
if status == 503:
burst_503 += 1
else:
print("unexpected status during postgres stop", status)
if burst_503 < 8:
raise SystemExit("FAIL expected every business write to be 503")
print("PASS business 503 while postgres stopped", burst_503)
compose("start", "postgres")
wait_ready_http()
print("PASS ready after postgres start")
print("-- collector stopped: API still accepts a registration --")
compose("stop", "otel-collector")
time.sleep(1)
status, _ = api.call(
"POST",
"/api/v1/users",
{
"username": "l6c_" + uuid.uuid4().hex[:10],
"display_name": "CollectorDown",
"password": "T3st-" + uuid.uuid4().hex,
},
)
expect(201, status, "register while collector is down")
compose("start", "otel-collector")
print("PASS collector restarted")
print("firing_at_end", ",".join(alerts()) or "-")
if __name__ == "__main__":
main()