90 lines
3.0 KiB
Python
90 lines
3.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Stop worker, build a real durable backlog, then verify queue-lag recovery."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from client import Client # noqa: E402
|
|
from slo_watch import snapshot # noqa: E402
|
|
|
|
|
|
FILES = ["-f", "compose.yaml"]
|
|
if (ROOT / "compose.local.yaml").exists():
|
|
FILES += ["-f", "compose.local.yaml"]
|
|
FILES += ["-f", "compose.telemetry.yaml"]
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(["docker", "compose", *FILES, "--env-file", ".env", *args],
|
|
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
|
|
|
|
|
def now():
|
|
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
|
|
|
|
def main():
|
|
client = Client("http://127.0.0.1:8080")
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
image = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
samples = []
|
|
stop = threading.Event()
|
|
|
|
def watch():
|
|
while not stop.is_set():
|
|
samples.append(snapshot("http://127.0.0.1:9090"))
|
|
stop.wait(5)
|
|
|
|
watcher = threading.Thread(target=watch, daemon=True)
|
|
watcher.start()
|
|
times = {"started": now()}
|
|
worker_stopped = False
|
|
try:
|
|
compose("stop", "worker")
|
|
worker_stopped = True
|
|
times["worker_stopped"] = now()
|
|
attachments = []
|
|
for _ in range(8):
|
|
response = client.call(
|
|
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename=queue-probe.png",
|
|
202, actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment",
|
|
)
|
|
attachments.append(response["json"])
|
|
times["uploads_accepted"] = now()
|
|
time.sleep(35)
|
|
compose("up", "-d", "worker")
|
|
worker_stopped = False
|
|
times["worker_started"] = now()
|
|
for item in attachments:
|
|
client.wait_ready(alice, item, wait_seconds=120)
|
|
times["all_ready"] = now()
|
|
time.sleep(40)
|
|
finally:
|
|
if worker_stopped:
|
|
compose("up", "-d", "worker")
|
|
stop.set()
|
|
watcher.join(timeout=6)
|
|
states = [item["alerts"].get("Lab6QueueLag") for item in samples]
|
|
data = {"times": times, "queued_uploads": len(attachments),
|
|
"ready_uploads": len(attachments), "prometheus_samples": samples,
|
|
"queue_alert_states": sorted(set(states))}
|
|
path = ROOT / "evidence/queue-worker-stop-2026-10-02.json"
|
|
path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps({"queued_uploads": len(attachments), "states": data["queue_alert_states"],
|
|
"times": times, "output": str(path)}, sort_keys=True))
|
|
if "firing" not in states or states[-1] != "inactive":
|
|
raise SystemExit("queue lag alert did not fire and resolve")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|