feat: add pipeline tracing and observability alerts
This commit is contained in:
@@ -31,7 +31,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals"])
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
@@ -47,13 +47,42 @@ def main():
|
||||
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
||||
print("Источник Grafana:", uid)
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
|
||||
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
if args.criterion == "pipeline":
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
client.wait_ready(bob, upload["json"])
|
||||
request_id = upload["headers"]["X-Request-Id"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=start, end=time.time_ns())["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
|
||||
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
|
||||
if producer and consumer:
|
||||
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
|
||||
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
|
||||
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
|
||||
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
|
||||
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
|
||||
"worker S3 and PostgreSQL spans missing")
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("upload trace did not include the worker pipeline")
|
||||
|
||||
marker = "telemetry-secret-" + uuid.uuid4().hex
|
||||
message = client.message(alice, chat, marker)
|
||||
request_id = message["headers"]["X-Request-Id"]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
def trace_for(request_id, matches):
|
||||
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
|
||||
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
|
||||
deadline = time.monotonic() + 35
|
||||
while time.monotonic() < deadline:
|
||||
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
|
||||
logs = json.load(response)["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
try:
|
||||
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
|
||||
trace = json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
else:
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
if matches(spans):
|
||||
return trace_id
|
||||
time.sleep(2)
|
||||
raise Failure("expected trace was not found")
|
||||
|
||||
def alerts(name):
|
||||
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
|
||||
with urllib.request.urlopen(endpoint, timeout=5) as response:
|
||||
active = json.load(response)["data"]["alerts"]
|
||||
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
|
||||
|
||||
def wait_alert(name, firing, exercise=lambda: None):
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
exercise()
|
||||
if alerts(name) == firing:
|
||||
print(name, "firing" if firing else "resolved", flush=True)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure(name + ": expected alert state not observed")
|
||||
|
||||
if args.incident == "worker":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
wait_alert("MessengerQueueLag", True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
client.wait_ready(alice, attachment)
|
||||
wait_alert("MessengerQueueLag", False)
|
||||
elif args.incident == "database":
|
||||
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
|
||||
marker = "lab6-fault-" + uuid.uuid4().hex
|
||||
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||||
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
|
||||
"sh", marker]
|
||||
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
time.sleep(2)
|
||||
|
||||
request_ids = []
|
||||
|
||||
def exercise():
|
||||
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
|
||||
request_ids.append(response["headers"]["X-Request-Id"])
|
||||
|
||||
wait_alert("MessengerHighErrorRate", True, exercise)
|
||||
finally:
|
||||
# Terminate only the synthetic lock session so the transaction rolls back immediately.
|
||||
compose("exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
|
||||
lock.wait(timeout=10)
|
||||
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
|
||||
trace_id = trace_for(request_ids[-1], lambda spans: any(
|
||||
span["name"] == "postgresql query"
|
||||
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
|
||||
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
|
||||
wait_alert("MessengerHighErrorRate", False)
|
||||
elif args.incident == "retry":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
attachment = upload["json"]
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 25
|
||||
while time.monotonic() < deadline:
|
||||
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
|
||||
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
|
||||
cwd=ROOT, text=True, timeout=10)
|
||||
if int(output.strip()) > 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("durable retry was not observed")
|
||||
finally:
|
||||
compose("up", "-d", "--wait", "s3")
|
||||
client.wait_ready(alice, attachment)
|
||||
def retried(spans):
|
||||
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
|
||||
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
|
||||
return len(set(attempts)) >= 2
|
||||
|
||||
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
|
||||
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
else:
|
||||
since = str(int(time.time()))
|
||||
compose("stop", "tempo")
|
||||
try:
|
||||
for _ in range(8):
|
||||
client.message(alice, chat, "telemetry-backend-off")
|
||||
time.sleep(5)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(alice, attachment)
|
||||
compose("stats", "--no-stream", "otel-collector")
|
||||
print("API и worker работали при остановленном Tempo.", flush=True)
|
||||
finally:
|
||||
compose("start", "tempo")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
|
||||
cwd=ROOT, text=True, timeout=15)
|
||||
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
|
||||
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
|
||||
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
|
||||
print("FAIL:", error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user