Files
mtusi2026lab6/scripts/observe.py
T

280 lines
9.8 KiB
Python

#!/usr/bin/env python3
"""Collect sanitized evidence from the local lab6 stand."""
import argparse
import json
import subprocess
import sys
import time
from pathlib import Path
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def compose(*args, **kwargs):
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
def api_json(port, path, **params):
url = f"http://127.0.0.1:{port}{path}"
if params:
url += "?" + urlencode(params)
with urlopen(url, timeout=5) as response:
return json.load(response)
def prom(query):
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
return float(data[0]["value"][1]) if data else 0
def until(check, timeout=45):
end = time.monotonic() + timeout
while time.monotonic() < end:
try:
result = check()
if result:
return result
except (HTTPError, URLError):
pass
time.sleep(0.5)
raise RuntimeError(f"Timed out waiting for {check.__name__}")
def log_for(request_id):
data = api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
for stream in data:
for value in stream["values"]:
return value[2] if len(value) > 2 else stream["stream"]
return None
def span_names(trace_id):
data = api_json(3206, "/api/traces/" + trace_id)
return [
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
for batch in data.get("batches", data.get("resourceSpans", []))
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
for span in group["spans"]
]
def alerts():
return {
a["labels"]["alertname"]: a["state"]
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
}
def correlation(client, actor, chat, retry=False):
if retry:
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
attachment = client.call(
"POST",
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
202,
actor=actor,
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"},
schema="Attachment",
)
if retry:
job_attachment = attachment["json"]["id"]
compose(
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-Xq",
"-v",
"ON_ERROR_STOP=1",
"-v",
"attachment=" + job_attachment,
input=(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
"WHERE attachment_id=:'attachment';\n"
),
text=True,
stdout=subprocess.DEVNULL,
)
finally:
if retry:
compose("start", "worker", stdout=subprocess.DEVNULL)
client.wait_ready(actor, attachment["json"])
request_id = attachment["headers"]["X-Request-Id"]
log = until(lambda: log_for(request_id))
trace_id = log["trace_id"]
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
def connected():
spans = span_names(trace_id)
names = {row[0] for row in spans}
consumers = sum(row[0] == "jobs.consume" for row in spans)
return spans if expected <= names and consumers >= (2 if retry else 1) else None
spans = until(connected)
by_id = {row[2]: row for row in spans}
consumers = [row for row in spans if row[0] == "jobs.consume"]
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
logs = until(
lambda: api_json(
3106,
"/loki/api/v1/query_range",
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
start=str(time.time_ns() - 180_000_000_000),
limit=5,
)["data"]["result"]
)
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
return {
"request_id": request_id,
"trace_id": trace_id,
"span_names": sorted({r[0] for r in spans}),
"worker_parent": by_id[consumers[0][3]][0],
"worker_attempts": len(consumers),
"api_and_worker_loki": True,
}
def worker_incident(client, actor, chat):
compose("stop", "worker", stdout=subprocess.DEVNULL)
try:
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
until(lambda: alerts().get("MessengerQueueLag") == "firing")
record = {
"depth_at_firing": prom("max(messenger_queue_depth)"),
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
}
assert record["depth_at_firing"] >= 3
finally:
compose("start", "worker", stdout=subprocess.DEVNULL)
for attachment in uploads:
client.wait_ready(actor, attachment)
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
def database_incident(client, actor):
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
command = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-U",
"messenger",
"-d",
"messenger",
"-XqAt",
"-v",
"ON_ERROR_STOP=1",
]
start = time.monotonic()
lock = subprocess.Popen(
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
)
lock.stdin.write(
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
)
lock.stdin.close()
try:
assert lock.stdout.readline().strip() == "locked"
attempts = []
while time.monotonic() - start < 34:
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
attempts.append(attempt)
states = alerts()
if all(
states.get(a) == "firing"
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
):
break
assert any(item["status"] == 503 for item in attempts)
active = alerts()
assert active.get("MessengerDatabaseErrors") == "firing", active
assert active.get("MessengerHttpErrors") == "firing", active
finally:
lock.wait(timeout=46)
lock.stdout.close()
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
# Keep successful traffic to publish fresh data while the alert window slides.
def resolved():
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
until(resolved)
failure = next(item for item in attempts if item["status"] == 503)
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
names = span_names(failure_log["trace_id"])
assert "db.read" in {row[0] for row in names}
return {
"statuses": [item["status"] for item in attempts],
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
"trace_id": failure_log["trace_id"],
"state": ["firing", "resolved"],
"spans": sorted({row[0] for row in names}),
}
def backend_incident(client, actor, chat):
lost = "sum(otelcol_exporter_send_failed_spans_total)"
baseline = prom(lost)
compose("stop", "tempo", stdout=subprocess.DEVNULL)
try:
for _ in range(15):
client.message(actor, chat, "synthetic telemetry outage probe")
time.sleep(0.5)
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
memory = prom("max(otelcol_process_memory_rss_bytes)")
assert memory > 0
finally:
compose("start", "tempo", stdout=subprocess.DEVNULL)
client.call("GET", "/health/ready", schema="Health")
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, required=True)
parser.add_argument("--incidents", action="store_true")
parser.add_argument("--retry", action="store_true")
args = parser.parse_args()
if args.output.exists():
parser.error("choose a new evidence file")
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
if args.incidents:
result["worker_stop"] = worker_incident(client, alice, chat)
print("PASS: queue alert firing and resolved", flush=True)
result["database_lock"] = database_incident(client, alice)
print("PASS: database and HTTP alerts firing and resolved", flush=True)
result["tempo_stop"] = backend_incident(client, alice, chat)
print("PASS: messages persisted during Tempo outage", flush=True)
args.output.write_text(json.dumps(result, indent=2) + "\n")
if __name__ == "__main__":
main()