forked from mtusi2026/mtusi2026lab6
feat: add pipeline tracing and observability alerts
This commit is contained in:
@@ -31,7 +31,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals"])
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
@@ -47,13 +47,42 @@ def main():
|
||||
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
||||
print("Источник Grafana:", uid)
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
|
||||
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
if args.criterion == "pipeline":
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
client.wait_ready(bob, upload["json"])
|
||||
request_id = upload["headers"]["X-Request-Id"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=start, end=time.time_ns())["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
|
||||
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
|
||||
if producer and consumer:
|
||||
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
|
||||
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
|
||||
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
|
||||
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
|
||||
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
|
||||
"worker S3 and PostgreSQL spans missing")
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("upload trace did not include the worker pipeline")
|
||||
|
||||
marker = "telemetry-secret-" + uuid.uuid4().hex
|
||||
message = client.message(alice, chat, marker)
|
||||
request_id = message["headers"]["X-Request-Id"]
|
||||
|
||||
Reference in New Issue
Block a user