forked from mtusi2026/mtusi2026lab6
280 lines
9.8 KiB
Python
280 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Collect sanitized evidence from the local lab6 stand."""
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import urlencode
|
|
from urllib.request import urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client # noqa: E402
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, **kwargs)
|
|
|
|
|
|
def api_json(port, path, **params):
|
|
url = f"http://127.0.0.1:{port}{path}"
|
|
if params:
|
|
url += "?" + urlencode(params)
|
|
with urlopen(url, timeout=5) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def prom(query):
|
|
data = api_json(9096, "/api/v1/query", query=query)["data"]["result"]
|
|
return float(data[0]["value"][1]) if data else 0
|
|
|
|
|
|
def until(check, timeout=45):
|
|
end = time.monotonic() + timeout
|
|
while time.monotonic() < end:
|
|
try:
|
|
result = check()
|
|
if result:
|
|
return result
|
|
except (HTTPError, URLError):
|
|
pass
|
|
time.sleep(0.5)
|
|
raise RuntimeError(f"Timed out waiting for {check.__name__}")
|
|
|
|
|
|
def log_for(request_id):
|
|
data = api_json(
|
|
3106,
|
|
"/loki/api/v1/query_range",
|
|
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
|
start=str(time.time_ns() - 180_000_000_000),
|
|
limit=5,
|
|
)["data"]["result"]
|
|
for stream in data:
|
|
for value in stream["values"]:
|
|
return value[2] if len(value) > 2 else stream["stream"]
|
|
return None
|
|
|
|
|
|
def span_names(trace_id):
|
|
data = api_json(3206, "/api/traces/" + trace_id)
|
|
return [
|
|
(span["name"], span.get("kind"), span["spanId"], span.get("parentSpanId"))
|
|
for batch in data.get("batches", data.get("resourceSpans", []))
|
|
for group in batch.get("scopeSpans", batch.get("instrumentationLibrarySpans", []))
|
|
for span in group["spans"]
|
|
]
|
|
|
|
|
|
def alerts():
|
|
return {
|
|
a["labels"]["alertname"]: a["state"]
|
|
for a in api_json(9096, "/api/v1/alerts")["data"]["alerts"]
|
|
}
|
|
|
|
|
|
def correlation(client, actor, chat, retry=False):
|
|
if retry:
|
|
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
|
try:
|
|
attachment = client.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat['id']}/attachments?filename=observability.png",
|
|
202,
|
|
actor=actor,
|
|
raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
if retry:
|
|
job_attachment = attachment["json"]["id"]
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-Xq",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-v",
|
|
"attachment=" + job_attachment,
|
|
input=(
|
|
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) "
|
|
"SELECT job_id,'before_effect','transient',1 FROM attachment_jobs "
|
|
"WHERE attachment_id=:'attachment';\n"
|
|
),
|
|
text=True,
|
|
stdout=subprocess.DEVNULL,
|
|
)
|
|
finally:
|
|
if retry:
|
|
compose("start", "worker", stdout=subprocess.DEVNULL)
|
|
client.wait_ready(actor, attachment["json"])
|
|
request_id = attachment["headers"]["X-Request-Id"]
|
|
log = until(lambda: log_for(request_id))
|
|
trace_id = log["trace_id"]
|
|
expected = {"jobs.publish", "jobs.consume", "s3.put", "s3.download", "db.transaction"}
|
|
|
|
def connected():
|
|
spans = span_names(trace_id)
|
|
names = {row[0] for row in spans}
|
|
consumers = sum(row[0] == "jobs.consume" for row in spans)
|
|
return spans if expected <= names and consumers >= (2 if retry else 1) else None
|
|
|
|
spans = until(connected)
|
|
by_id = {row[2]: row for row in spans}
|
|
consumers = [row for row in spans if row[0] == "jobs.consume"]
|
|
assert all(by_id[row[3]][0] == "jobs.publish" for row in consumers)
|
|
logs = until(
|
|
lambda: api_json(
|
|
3106,
|
|
"/loki/api/v1/query_range",
|
|
query='{service_name="messenger-worker"} | trace_id="' + trace_id + '"',
|
|
start=str(time.time_ns() - 180_000_000_000),
|
|
limit=5,
|
|
)["data"]["result"]
|
|
)
|
|
assert any(value[1] == "job_processed" for row in logs for value in row["values"])
|
|
return {
|
|
"request_id": request_id,
|
|
"trace_id": trace_id,
|
|
"span_names": sorted({r[0] for r in spans}),
|
|
"worker_parent": by_id[consumers[0][3]][0],
|
|
"worker_attempts": len(consumers),
|
|
"api_and_worker_loki": True,
|
|
}
|
|
|
|
|
|
def worker_incident(client, actor, chat):
|
|
compose("stop", "worker", stdout=subprocess.DEVNULL)
|
|
try:
|
|
uploads = [client.attachment(actor, chat)[0] for _ in range(3)]
|
|
until(lambda: alerts().get("MessengerQueueLag") == "firing")
|
|
record = {
|
|
"depth_at_firing": prom("max(messenger_queue_depth)"),
|
|
"oldest_seconds_at_firing": prom("max(messenger_queue_oldest_seconds)"),
|
|
}
|
|
assert record["depth_at_firing"] >= 3
|
|
finally:
|
|
compose("start", "worker", stdout=subprocess.DEVNULL)
|
|
for attachment in uploads:
|
|
client.wait_ready(actor, attachment)
|
|
until(lambda: not prom("max(messenger_queue_depth)") and "MessengerQueueLag" not in alerts())
|
|
return {**record, "state": ["firing", "resolved"], "uploads_ready": len(uploads)}
|
|
|
|
|
|
def database_incident(client, actor):
|
|
# Independent, bounded transaction. Lock sessions only; readiness can still use the schema.
|
|
command = [
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-U",
|
|
"messenger",
|
|
"-d",
|
|
"messenger",
|
|
"-XqAt",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
]
|
|
start = time.monotonic()
|
|
lock = subprocess.Popen(
|
|
command, cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True
|
|
)
|
|
lock.stdin.write(
|
|
"BEGIN;\nLOCK TABLE sessions IN ACCESS EXCLUSIVE MODE;\n"
|
|
"\\echo locked\nSELECT pg_sleep(42);\nROLLBACK;\n"
|
|
)
|
|
lock.stdin.close()
|
|
try:
|
|
assert lock.stdout.readline().strip() == "locked"
|
|
attempts = []
|
|
while time.monotonic() - start < 34:
|
|
attempt = client.call("GET", "/api/v1/users/me", [200, 503], actor=actor, schema="User")
|
|
attempts.append(attempt)
|
|
states = alerts()
|
|
if all(
|
|
states.get(a) == "firing"
|
|
for a in ("MessengerHttpErrors", "MessengerDatabaseErrors")
|
|
):
|
|
break
|
|
assert any(item["status"] == 503 for item in attempts)
|
|
active = alerts()
|
|
assert active.get("MessengerDatabaseErrors") == "firing", active
|
|
assert active.get("MessengerHttpErrors") == "firing", active
|
|
finally:
|
|
lock.wait(timeout=46)
|
|
lock.stdout.close()
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
|
|
# Keep successful traffic to publish fresh data while the alert window slides.
|
|
def resolved():
|
|
client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
|
return not ({"MessengerDatabaseErrors", "MessengerHttpErrors"} & alerts().keys())
|
|
|
|
until(resolved)
|
|
failure = next(item for item in attempts if item["status"] == 503)
|
|
failure_log = until(lambda: log_for(failure["headers"]["X-Request-Id"]))
|
|
names = span_names(failure_log["trace_id"])
|
|
assert "db.read" in {row[0] for row in names}
|
|
return {
|
|
"statuses": [item["status"] for item in attempts],
|
|
"max_latency_ms": max(item["elapsed"] for item in attempts) * 1000,
|
|
"trace_id": failure_log["trace_id"],
|
|
"state": ["firing", "resolved"],
|
|
"spans": sorted({row[0] for row in names}),
|
|
}
|
|
|
|
|
|
def backend_incident(client, actor, chat):
|
|
lost = "sum(otelcol_exporter_send_failed_spans_total)"
|
|
baseline = prom(lost)
|
|
compose("stop", "tempo", stdout=subprocess.DEVNULL)
|
|
try:
|
|
for _ in range(15):
|
|
client.message(actor, chat, "synthetic telemetry outage probe")
|
|
time.sleep(0.5)
|
|
delta = until(lambda: value if (value := prom(lost) - baseline) > 0 else None)
|
|
memory = prom("max(otelcol_process_memory_rss_bytes)")
|
|
assert memory > 0
|
|
finally:
|
|
compose("start", "tempo", stdout=subprocess.DEVNULL)
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
return {"successful_messages": 15, "failed_export_spans": delta, "collector_rss_bytes": memory}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
parser.add_argument("--incidents", action="store_true")
|
|
parser.add_argument("--retry", action="store_true")
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
parser.error("choose a new evidence file")
|
|
client = Client(args.base_url)
|
|
alice, _, _, chat = client.bootstrap()
|
|
result = {"correlation": correlation(client, alice, chat, retry=args.retry)}
|
|
print("PASS: same Tempo trace and Loki logs for upload and worker", flush=True)
|
|
if args.incidents:
|
|
result["worker_stop"] = worker_incident(client, alice, chat)
|
|
print("PASS: queue alert firing and resolved", flush=True)
|
|
result["database_lock"] = database_incident(client, alice)
|
|
print("PASS: database and HTTP alerts firing and resolved", flush=True)
|
|
result["tempo_stop"] = backend_incident(client, alice, chat)
|
|
print("PASS: messages persisted during Tempo outage", flush=True)
|
|
args.output.write_text(json.dumps(result, indent=2) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|