Files
mtusi2026lab6/scripts/check_observability_incidents.py
T

172 lines
8.2 KiB
Python

#!/usr/bin/env python3
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
def trace_for(request_id, matches):
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
deadline = time.monotonic() + 35
while time.monotonic() < deadline:
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
logs = json.load(response)["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
try:
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
trace = json.load(response)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
else:
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
if matches(spans):
return trace_id
time.sleep(2)
raise Failure("expected trace was not found")
def alerts(name):
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
with urllib.request.urlopen(endpoint, timeout=5) as response:
active = json.load(response)["data"]["alerts"]
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
def wait_alert(name, firing, exercise=lambda: None):
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
exercise()
if alerts(name) == firing:
print(name, "firing" if firing else "resolved", flush=True)
return
time.sleep(2)
raise Failure(name + ": expected alert state not observed")
if args.incident == "worker":
compose("stop", "worker")
try:
attachment, _ = client.attachment(alice, chat)
wait_alert("MessengerQueueLag", True)
finally:
compose("start", "worker")
client.wait_ready(alice, attachment)
wait_alert("MessengerQueueLag", False)
elif args.incident == "database":
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
marker = "lab6-fault-" + uuid.uuid4().hex
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
"sh", marker]
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
try:
time.sleep(2)
request_ids = []
def exercise():
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
request_ids.append(response["headers"]["X-Request-Id"])
wait_alert("MessengerHighErrorRate", True, exercise)
finally:
# Terminate only the synthetic lock session so the transaction rolls back immediately.
compose("exec", "-T", "db", "sh", "-c",
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
lock.wait(timeout=10)
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
trace_id = trace_for(request_ids[-1], lambda spans: any(
span["name"] == "postgresql query"
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
wait_alert("MessengerHighErrorRate", False)
elif args.incident == "retry":
compose("stop", "worker")
try:
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
attachment = upload["json"]
compose("stop", "s3")
try:
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
output = subprocess.check_output(
["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
cwd=ROOT, text=True, timeout=10)
if int(output.strip()) > 0:
break
time.sleep(1)
else:
raise Failure("durable retry was not observed")
finally:
compose("up", "-d", "--wait", "s3")
client.wait_ready(alice, attachment)
def retried(spans):
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
return len(set(attempts)) >= 2
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
finally:
compose("start", "worker")
else:
since = str(int(time.time()))
compose("stop", "tempo")
try:
for _ in range(8):
client.message(alice, chat, "telemetry-backend-off")
time.sleep(5)
attachment, _ = client.attachment(alice, chat)
client.wait_ready(alice, attachment)
compose("stats", "--no-stream", "otel-collector")
print("API и worker работали при остановленном Tempo.", flush=True)
finally:
compose("start", "tempo")
output = subprocess.check_output(
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
cwd=ROOT, text=True, timeout=15)
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
print("FAIL:", error, file=sys.stderr)
sys.exit(1)