Files
mtusi2026lab6/student-tests/test_queue_incident.py
T

90 lines
3.0 KiB
Python

#!/usr/bin/env python3
"""Stop worker, build a real durable backlog, then verify queue-lag recovery."""
import json
import subprocess
import sys
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client # noqa: E402
from slo_watch import snapshot # noqa: E402
FILES = ["-f", "compose.yaml"]
if (ROOT / "compose.local.yaml").exists():
FILES += ["-f", "compose.local.yaml"]
FILES += ["-f", "compose.telemetry.yaml"]
def compose(*args):
subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
def now():
return datetime.now(timezone.utc).isoformat(timespec="seconds")
def main():
client = Client("http://127.0.0.1:8080")
alice, _bob, _eve, chat = client.bootstrap()
image = (ROOT / "tests/fixtures/sample.png").read_bytes()
samples = []
stop = threading.Event()
def watch():
while not stop.is_set():
samples.append(snapshot("http://127.0.0.1:9090"))
stop.wait(5)
watcher = threading.Thread(target=watch, daemon=True)
watcher.start()
times = {"started": now()}
worker_stopped = False
try:
compose("stop", "worker")
worker_stopped = True
times["worker_stopped"] = now()
attachments = []
for _ in range(8):
response = client.call(
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png",
202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment",
)
attachments.append(response["json"])
times["uploads_accepted"] = now()
time.sleep(35)
compose("up", "-d", "worker")
worker_stopped = False
times["worker_started"] = now()
for item in attachments:
client.wait_ready(alice, item, wait_seconds=120)
times["all_ready"] = now()
time.sleep(40)
finally:
if worker_stopped:
compose("up", "-d", "worker")
stop.set()
watcher.join(timeout=6)
states = [item["alerts"].get("Lab6QueueLag") for item in samples]
data = {"times": times, "queued_uploads": len(attachments),
"ready_uploads": len(attachments), "prometheus_samples": samples,
"queue_alert_states": sorted(set(states))}
path = ROOT / "evidence/queue-worker-stop-2026-10-02.json"
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"],
"times": times, "output": str(path)}, sort_keys=True))
if "firing" not in states or states[-1] != "inactive":
raise SystemExit("queue lag alert did not fire and resolve")
if __name__ == "__main__":
main()