feat: add pipeline tracing and observability alerts

This commit is contained in:
Dmitrii Grishko
2026-11-13 18:52:25 +03:00
parent 549edc8d7a
commit d753e348eb
19 changed files with 821 additions and 21 deletions
+31 -2
View File
@@ -31,7 +31,7 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
args = parser.parse_args()
def query(path, **params):
@@ -47,13 +47,42 @@ def main():
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
if args.criterion == "pipeline":
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
client.wait_ready(bob, upload["json"])
request_id = upload["headers"]["X-Request-Id"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=start, end=time.time_ns())["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
trace = source("tempo", "/api/traces/" + trace_id)
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
if producer and consumer:
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
"worker S3 and PostgreSQL spans missing")
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
return
time.sleep(2)
raise Failure("upload trace did not include the worker pipeline")
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]