feat: add pipeline tracing and observability alerts

This commit is contained in:
Dmitrii Grishko
2026-11-13 18:52:25 +03:00
parent 549edc8d7a
commit d753e348eb
19 changed files with 821 additions and 21 deletions
+31 -2
View File
@@ -31,7 +31,7 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
args = parser.parse_args()
def query(path, **params):
@@ -47,13 +47,42 @@ def main():
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
if args.criterion == "pipeline":
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
client.wait_ready(bob, upload["json"])
request_id = upload["headers"]["X-Request-Id"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=start, end=time.time_ns())["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
trace = source("tempo", "/api/traces/" + trace_id)
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
if producer and consumer:
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
"worker S3 and PostgreSQL spans missing")
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
return
time.sleep(2)
raise Failure("upload trace did not include the worker pipeline")
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
def trace_for(request_id, matches):
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
deadline = time.monotonic() + 35
while time.monotonic() < deadline:
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
logs = json.load(response)["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
try:
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
trace = json.load(response)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
else:
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
if matches(spans):
return trace_id
time.sleep(2)
raise Failure("expected trace was not found")
def alerts(name):
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
with urllib.request.urlopen(endpoint, timeout=5) as response:
active = json.load(response)["data"]["alerts"]
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
def wait_alert(name, firing, exercise=lambda: None):
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
exercise()
if alerts(name) == firing:
print(name, "firing" if firing else "resolved", flush=True)
return
time.sleep(2)
raise Failure(name + ": expected alert state not observed")
if args.incident == "worker":
compose("stop", "worker")
try:
attachment, _ = client.attachment(alice, chat)
wait_alert("MessengerQueueLag", True)
finally:
compose("start", "worker")
client.wait_ready(alice, attachment)
wait_alert("MessengerQueueLag", False)
elif args.incident == "database":
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
marker = "lab6-fault-" + uuid.uuid4().hex
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
"sh", marker]
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
try:
time.sleep(2)
request_ids = []
def exercise():
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
request_ids.append(response["headers"]["X-Request-Id"])
wait_alert("MessengerHighErrorRate", True, exercise)
finally:
# Terminate only the synthetic lock session so the transaction rolls back immediately.
compose("exec", "-T", "db", "sh", "-c",
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
lock.wait(timeout=10)
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
trace_id = trace_for(request_ids[-1], lambda spans: any(
span["name"] == "postgresql query"
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
wait_alert("MessengerHighErrorRate", False)
elif args.incident == "retry":
compose("stop", "worker")
try:
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
attachment = upload["json"]
compose("stop", "s3")
try:
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
output = subprocess.check_output(
["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
cwd=ROOT, text=True, timeout=10)
if int(output.strip()) > 0:
break
time.sleep(1)
else:
raise Failure("durable retry was not observed")
finally:
compose("up", "-d", "--wait", "s3")
client.wait_ready(alice, attachment)
def retried(spans):
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
return len(set(attempts)) >= 2
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
finally:
compose("start", "worker")
else:
since = str(int(time.time()))
compose("stop", "tempo")
try:
for _ in range(8):
client.message(alice, chat, "telemetry-backend-off")
time.sleep(5)
attachment, _ = client.attachment(alice, chat)
client.wait_ready(alice, attachment)
compose("stats", "--no-stream", "otel-collector")
print("API и worker работали при остановленном Tempo.", flush=True)
finally:
compose("start", "tempo")
output = subprocess.check_output(
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
cwd=ROOT, text=True, timeout=15)
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
print("FAIL:", error, file=sys.stderr)
sys.exit(1)