|
|
|
@@ -0,0 +1,222 @@
|
|
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
"""Lab6 failure scenarios. Prints statuses, ids and metric values. Never tokens."""
|
|
|
|
|
|
|
|
|
|
import base64
|
|
|
|
|
import json
|
|
|
|
|
import os
|
|
|
|
|
import subprocess
|
|
|
|
|
import time
|
|
|
|
|
import urllib.error
|
|
|
|
|
import urllib.parse
|
|
|
|
|
import urllib.request
|
|
|
|
|
import uuid
|
|
|
|
|
from pathlib import Path
|
|
|
|
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
|
|
|
|
PROM = os.environ.get("PROMETHEUS_URL", "http://127.0.0.1:9090")
|
|
|
|
|
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def compose(*args: str) -> None:
|
|
|
|
|
command = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
|
|
|
|
|
if os.environ.get("COMPOSE_ALTPORTS") == "1":
|
|
|
|
|
command.extend(["-f", "compose.altports.yaml"])
|
|
|
|
|
subprocess.run([*command, *args], cwd=ROOT, check=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class Api:
|
|
|
|
|
def __init__(self) -> None:
|
|
|
|
|
self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
|
|
|
|
|
|
|
|
|
def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None, timeout=15):
|
|
|
|
|
hdrs = dict(headers or {})
|
|
|
|
|
data = raw
|
|
|
|
|
if body is not None:
|
|
|
|
|
data = json.dumps(body).encode("utf-8")
|
|
|
|
|
hdrs["Content-Type"] = "application/json"
|
|
|
|
|
if content_type:
|
|
|
|
|
hdrs["Content-Type"] = content_type
|
|
|
|
|
if token:
|
|
|
|
|
hdrs["Authorization"] = "Bearer " + token
|
|
|
|
|
request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method)
|
|
|
|
|
try:
|
|
|
|
|
response = self.opener.open(request, timeout=timeout)
|
|
|
|
|
except urllib.error.HTTPError as error:
|
|
|
|
|
response = error
|
|
|
|
|
with response:
|
|
|
|
|
payload = response.read()
|
|
|
|
|
return response.status, payload
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def expect(status, got, label):
|
|
|
|
|
if got != status:
|
|
|
|
|
raise SystemExit("FAIL %s expected %s got %s" % (label, status, got))
|
|
|
|
|
print("PASS", label, got)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def prom_query(expr: str):
|
|
|
|
|
url = PROM + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
|
|
|
|
with urllib.request.urlopen(url, timeout=10) as response:
|
|
|
|
|
body = json.loads(response.read().decode())
|
|
|
|
|
if body.get("status") != "success":
|
|
|
|
|
raise SystemExit("FAIL prometheus query")
|
|
|
|
|
result = body["data"]["result"]
|
|
|
|
|
if not result:
|
|
|
|
|
return None
|
|
|
|
|
return float(result[0]["value"][1])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def alerts():
|
|
|
|
|
with urllib.request.urlopen(PROM + "/api/v1/alerts", timeout=10) as response:
|
|
|
|
|
body = json.loads(response.read().decode())
|
|
|
|
|
names = []
|
|
|
|
|
for item in body["data"]["alerts"]:
|
|
|
|
|
if item.get("state") == "firing":
|
|
|
|
|
names.append(item["labels"].get("alertname", ""))
|
|
|
|
|
return sorted(names)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def wait_ready_http():
|
|
|
|
|
deadline = time.monotonic() + 90
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
try:
|
|
|
|
|
status, _ = Api().call("GET", "/health/ready", timeout=3)
|
|
|
|
|
if status == 200:
|
|
|
|
|
return
|
|
|
|
|
except Exception:
|
|
|
|
|
pass
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
raise SystemExit("FAIL API did not become ready")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def user(api: Api):
|
|
|
|
|
username = "l6_" + uuid.uuid4().hex[:10]
|
|
|
|
|
password = "T3st-" + uuid.uuid4().hex
|
|
|
|
|
status, _ = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/users",
|
|
|
|
|
{"username": username, "display_name": "Lab6", "password": password},
|
|
|
|
|
)
|
|
|
|
|
expect(201, status, "register")
|
|
|
|
|
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
|
|
|
|
status, payload = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
|
|
|
|
expect(201, status, "login")
|
|
|
|
|
return json.loads(payload)["token"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def main() -> None:
|
|
|
|
|
api = Api()
|
|
|
|
|
token = user(api)
|
|
|
|
|
status, chat_raw = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/chats",
|
|
|
|
|
{"title": "lab6-incidents", "member_ids": []},
|
|
|
|
|
token=token,
|
|
|
|
|
)
|
|
|
|
|
# member_ids must include someone? empty may 400. Create with only self via API rules.
|
|
|
|
|
if status != 201:
|
|
|
|
|
me_status, me_raw = api.call("GET", "/api/v1/users/me", token=token)
|
|
|
|
|
expect(200, me_status, "me")
|
|
|
|
|
me = json.loads(me_raw)["id"]
|
|
|
|
|
status, chat_raw = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/chats",
|
|
|
|
|
{"title": "lab6-incidents", "member_ids": [me]},
|
|
|
|
|
token=token,
|
|
|
|
|
)
|
|
|
|
|
expect(201, status, "chat")
|
|
|
|
|
chat_id = json.loads(chat_raw)["id"]
|
|
|
|
|
print("chat", chat_id)
|
|
|
|
|
|
|
|
|
|
print("-- worker stopped: queue age grows, content stays 409 --")
|
|
|
|
|
compose("stop", "worker")
|
|
|
|
|
content = FIXTURE.read_bytes()
|
|
|
|
|
status, attachment_raw = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/chats/%s/attachments?filename=sample.png" % chat_id,
|
|
|
|
|
token=token,
|
|
|
|
|
raw=content,
|
|
|
|
|
content_type="image/png",
|
|
|
|
|
)
|
|
|
|
|
expect(202, status, "upload while worker stopped")
|
|
|
|
|
attachment_id = json.loads(attachment_raw)["id"]
|
|
|
|
|
print("attachment", attachment_id)
|
|
|
|
|
status, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment_id, token=token)
|
|
|
|
|
expect(409, status, "content before ready")
|
|
|
|
|
deadline = time.monotonic() + 80
|
|
|
|
|
age = None
|
|
|
|
|
while time.monotonic() < deadline:
|
|
|
|
|
age = prom_query("max(messenger_queue_oldest_age_seconds)")
|
|
|
|
|
print("queue_oldest_age_seconds", age, "firing", ",".join(alerts()) or "-")
|
|
|
|
|
if age is not None and age >= 15:
|
|
|
|
|
break
|
|
|
|
|
time.sleep(5)
|
|
|
|
|
if age is None or age < 5:
|
|
|
|
|
raise SystemExit("FAIL queue age did not grow")
|
|
|
|
|
compose("start", "worker")
|
|
|
|
|
ready_deadline = time.monotonic() + 90
|
|
|
|
|
while time.monotonic() < ready_deadline:
|
|
|
|
|
status, body = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token)
|
|
|
|
|
expect(200, status, "poll")
|
|
|
|
|
parsed = json.loads(body)
|
|
|
|
|
if parsed["status"] == "ready":
|
|
|
|
|
print("PASS attachment ready after worker start", attachment_id)
|
|
|
|
|
break
|
|
|
|
|
if parsed["status"] == "failed":
|
|
|
|
|
raise SystemExit("FAIL attachment failed")
|
|
|
|
|
time.sleep(0.5)
|
|
|
|
|
else:
|
|
|
|
|
raise SystemExit("FAIL attachment did not become ready")
|
|
|
|
|
time.sleep(12)
|
|
|
|
|
print("queue_oldest_age_seconds_after", prom_query("max(messenger_queue_oldest_age_seconds)"))
|
|
|
|
|
|
|
|
|
|
print("-- postgres stopped: live stays 200, ready becomes 503 --")
|
|
|
|
|
compose("stop", "postgres")
|
|
|
|
|
time.sleep(2)
|
|
|
|
|
status, _ = api.call("GET", "/health/live")
|
|
|
|
|
expect(200, status, "live while postgres stopped")
|
|
|
|
|
status, _ = api.call("GET", "/health/ready")
|
|
|
|
|
expect(503, status, "ready while postgres stopped")
|
|
|
|
|
burst_503 = 0
|
|
|
|
|
for _ in range(8):
|
|
|
|
|
status, _ = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/users",
|
|
|
|
|
{
|
|
|
|
|
"username": "l6d_" + uuid.uuid4().hex[:8],
|
|
|
|
|
"display_name": "DbDown",
|
|
|
|
|
"password": "T3st-" + uuid.uuid4().hex,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
if status == 503:
|
|
|
|
|
burst_503 += 1
|
|
|
|
|
else:
|
|
|
|
|
print("unexpected status during postgres stop", status)
|
|
|
|
|
if burst_503 < 8:
|
|
|
|
|
raise SystemExit("FAIL expected every business write to be 503")
|
|
|
|
|
print("PASS business 503 while postgres stopped", burst_503)
|
|
|
|
|
compose("start", "postgres")
|
|
|
|
|
wait_ready_http()
|
|
|
|
|
print("PASS ready after postgres start")
|
|
|
|
|
|
|
|
|
|
print("-- collector stopped: API still accepts a registration --")
|
|
|
|
|
compose("stop", "otel-collector")
|
|
|
|
|
time.sleep(1)
|
|
|
|
|
status, _ = api.call(
|
|
|
|
|
"POST",
|
|
|
|
|
"/api/v1/users",
|
|
|
|
|
{
|
|
|
|
|
"username": "l6c_" + uuid.uuid4().hex[:10],
|
|
|
|
|
"display_name": "CollectorDown",
|
|
|
|
|
"password": "T3st-" + uuid.uuid4().hex,
|
|
|
|
|
},
|
|
|
|
|
)
|
|
|
|
|
expect(201, status, "register while collector is down")
|
|
|
|
|
compose("start", "otel-collector")
|
|
|
|
|
print("PASS collector restarted")
|
|
|
|
|
print("firing_at_end", ",".join(alerts()) or "-")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
|
main()
|