"""Lab6: связь запроса, структурного лога и трассы; сквозная трасса upload -> worker, включая retry. python student-tests/test_telemetry.py # все сценарии python student-tests/test_telemetry.py upload-chain # один (список SCENARIOS ниже) Нужен запущенный стенд (compose.yaml, FAULT_INJECTION=1 для retry/replay). Клиент сам задаёт W3C traceparent запроса, поэтому трасса находится в Tempo по известному trace_id, без поиска «похожей». Тест печатает дерево спанов (без данных пользователя) — это доказательство для отчёта. Сценарии: request уровень 3: POST сообщения -> server-спан API с дочерними спанами PostgreSQL/Redis, запись лога event=http.request с тем же trace_id/span_id и request_id из X-Request-Id; текст сообщения, пароль и токен не попали ни в логи (Loki), ни в атрибуты спанов. upload-chain уровень 4: одна трасса API(server) -> create(producer, outbox) -> dispatcher publish (producer) -> worker process (consumer) -> S3 PutObject x4 и COMMIT ready; логи worker с тем же trace_id. retry-chain уровень 4: временная ошибка S3 в worker (fault worker_s3_error) -> в той же трассе попытка 1 (ERROR, outcome=retry) -> новая публикация -> попытка 2 (ready). replay-link уровень 4: попытки исчерпаны -> failed -> ops.py replay создаёт НОВУЮ трассу со ссылкой (link) на исходную загрузку, worker завершает её в ready. history уровень 3: после restart prometheus/loki/tempo (volumes сохранены) прежние трасса, логи и метрика по-прежнему находятся. """ import json import subprocess import sys import time import uuid import obs from common import ROOT, call, check, compose, new_chat, new_user, run_tests, wait_attachment from images import png LOKI_WAIT = 60 TRACE_WAIT = 60 def upload_traced(user, chat_id, data, traceparent, filename="trace.png"): reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", 202, raw=data, token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent}) return reply def by_name(spans, name, service=None): return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)] def parent_of(spans, span): return next((s for s in spans if s["span_id"] == span["parent_id"]), None) def fault(point, count=1): for n in range(count): subprocess.run(["docker", "compose", "exec", "-T", "worker", "sh", "-c", f"touch /faults/{point}.{n + 1}"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) def ops(*args): result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], cwd=ROOT, capture_output=True, check=True) return json.loads(result.stdout.decode("utf-8")) # ---------- сценарии ---------- def test_request(): user = new_user("trace") chat = new_chat(user, title="Trace chat") marker = f"secret-text-{uuid.uuid4().hex}" trace_id, traceparent = obs.new_traceparent() reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": marker}, headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) request_id = reply.headers["x-request-id"] check(reply.headers.get("x-trace-id") == trace_id, "X-Trace-Id must continue the client's traceparent") print(f"POST message: 201, X-Request-Id={request_id}, trace_id={trace_id} (client traceparent)") spans = obs.wait_trace(trace_id, lambda s: by_name(s, "POST /api/v1/chats/{chat_id}/messages", "messenger-api") and any(x["kind"] == "CLIENT" and x["name"] == "INSERT" for x in s), TRACE_WAIT) server = by_name(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")[0] check(server["kind"] == "SERVER" and server["attrs"].get("http.response.status_code") == 201, "server span") check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span") db = [s for s in spans if s["parent_id"] == server["span_id"] and "db.statement" in s["attrs"] or "db.query.text" in s["attrs"]] check(db, "PostgreSQL client spans under the server span") check(by_name(spans, "redis GET", "messenger-api"), "Redis session lookup span") print(obs.span_tree(spans)) rows = obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}" | event="http.request"', 1, LOKI_WAIT) labels, line = rows[0] check(labels.get("request_id") == request_id, "log request_id == X-Request-Id") check(labels.get("span_id") == server["span_id"], "log span_id == server span id") for field in ("service_name", "severity_text", "event", "trace_id", "span_id"): check(labels.get(field), f"log field {field} is missing") print(f"Loki: {line!r} event={labels['event']} severity={labels['severity_text']} " f"service={labels['service_name']} trace_id={labels['trace_id']} span_id={labels['span_id']} " f"request_id={labels['request_id']}") # Ни текст сообщения, ни пароль, ни токен не попали в телеметрию. for secret, what in ((marker, "message text"), (user["password"], "password"), (user["token"][:24], "token")): found = obs.loki_query(f'{{service_name=~"messenger-.*"}} |= `{secret}`', minutes=10, limit=5) check(not found, f"{what} found in Loki logs") check(not any(secret in str(v) for s in spans for v in s["attrs"].values()), f"{what} found in span attributes") check(not any(k.lower().startswith(("http.request.header.authorization", "enduser")) for s in spans for k in s["attrs"]), "Authorization/enduser attributes in spans") print("PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id " "and request_id; no message text, password or token in logs/span attributes") def test_upload_chain(): user = new_user("chain") chat = new_chat(user, title="Chain chat") trace_id, traceparent = obs.new_traceparent() accepted = upload_traced(user, chat["id"], png(320, 200), traceparent).json ready = wait_attachment(user, accepted["id"]) check(ready["status"] == "ready", f"attachment is {ready['status']}") def complete(spans): # Ждём span'ы всех трёх процессов: каждый выгружает свой batch (до OTEL_BSP_SCHEDULE_DELAY, 5 с) # независимо, и в Docker batch worker нередко приходит в Tempo раньше batch API. return (by_name(spans, "commit result", "messenger-worker") and by_name(spans, "cleanup staging") and by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api") and by_name(spans, "create attachments", "messenger-api")) spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) print(obs.span_tree(spans)) server = by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api")[0] create = by_name(spans, "create attachments", "messenger-api")[0] publish = by_name(spans, "publish attachments", "messenger-dispatcher")[0] consume = by_name(spans, "process attachments.process.v1", "messenger-worker")[0] check(create["kind"] == "PRODUCER" and publish["kind"] == "PRODUCER" and consume["kind"] == "CONSUMER", "kinds") check(parent_of(spans, create)["span_id"] == server["span_id"], "create is a child of the API request") check(publish["parent_id"] == create["span_id"], "publish (dispatcher) is a child of create (API outbox)") check(consume["parent_id"] == publish["span_id"], "consume (worker) is a child of publish (W3C via AMQP headers)") check(consume["attrs"].get("messenger.job.outcome") == "ready", "consumer outcome ready") puts = [s for s in by_name(spans, "S3 PutObject", "messenger-worker") if s["attrs"].get("aws.s3.key", "").startswith(f"attachments/{accepted['id']}/v1/")] check(len(puts) == 4, f"4 PUTs of the final objects, got {len(puts)}") check({s["service"] for s in spans} >= {"messenger-api", "messenger-dispatcher", "messenger-worker"}, "services") events = {labels["event"] for labels, _ in obs.wait_logs( f'{{service_name="messenger-worker"}} | trace_id="{trace_id}"', 2, LOKI_WAIT)} check({"job.claimed", "job.ready"} <= events, f"worker logs with this trace_id: {events}") print(f"worker logs with trace_id {trace_id}: {sorted(events)}") print(f"PASS: one trace {trace_id}: API 202 -> outbox create -> dispatcher publish -> worker consume -> " f"4 S3 objects + COMMIT ready ({len(spans)} spans, services: " f"{', '.join(sorted({s['service'] for s in spans}))})") def test_retry_chain(): user = new_user("retry") chat = new_chat(user, title="Retry chat") before = obs.prom_value('sum(messenger_job_duration_seconds_count{outcome="retry"})', 0.0) fault("worker_s3_error", 1) trace_id, traceparent = obs.new_traceparent() accepted = upload_traced(user, chat["id"], png(64, 48), traceparent).json ready = wait_attachment(user, accepted["id"]) check(ready["status"] == "ready", f"attachment is {ready['status']}") def complete(spans): # + обе публикации dispatcher: его batch выгружается независимо от batch worker (см. test_upload_chain) return (len(by_name(spans, "process attachments.process.v1")) >= 2 and by_name(spans, "commit result") and len(by_name(spans, "publish attachments")) >= 2) spans = obs.wait_trace(trace_id, complete, TRACE_WAIT) print(obs.span_tree(spans)) attempts = sorted(by_name(spans, "process attachments.process.v1", "messenger-worker"), key=lambda s: s["start"]) first, second = attempts[0], attempts[-1] check(first["status"] == "ERROR" and first["attrs"].get("messenger.job.outcome") == "retry", "attempt 1 = retry") check(first["attrs"].get("error.type") == "storage_unavailable", "attempt 1 error.type") check(second["attrs"].get("messenger.job.outcome") == "ready", "attempt 2 = ready") retry_create = [s for s in by_name(spans, "create attachments") if s["parent_id"] == first["span_id"]] check(retry_create, "retry message is created inside attempt 1 (outbox in the same transaction)") second_publish = parent_of(spans, second) check(second_publish["name"] == "publish attachments" and second_publish["parent_id"] == retry_create[0]["span_id"], "attempt 2 consumes the retry publication") check(len(by_name(spans, "publish attachments")) == 2, "two publications in one trace") rows = obs.wait_logs(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="job.retry"', 1, LOKI_WAIT) print(f"worker log: {rows[0][1]!r} error_type={rows[0][0].get('error_type')}") after = obs.wait_metric('sum(messenger_job_duration_seconds_count{outcome="retry"})', lambda v: v > before, 40) print(f"messenger_job_duration_seconds_count{{outcome=\"retry\"}}: {before:g} -> {after:g}") print(f"PASS: retry in one trace {trace_id}: attempt 1 ERROR storage_unavailable -> create -> publish -> " f"attempt 2 ready; retry counted in metrics and logged") def test_replay_link(): user = new_user("replay") chat = new_chat(user, title="Replay chat") fault("worker_s3_error", 5) trace_id, traceparent = obs.new_traceparent() accepted = upload_traced(user, chat["id"], png(32, 32), traceparent).json failed = wait_attachment(user, accepted["id"], statuses=("failed",), seconds=120) check(failed["error_code"] == "retry_exhausted", f"error_code {failed['error_code']}") job = next(j for j in ops("failed", "--limit", "20") if j["attachment_id"] == accepted["id"]) result = ops("replay", job["job_id"], "--reason", "lab6 link test") check("replayed" in result, f"replay refused: {result}") ready = wait_attachment(user, accepted["id"]) check(ready["status"] == "ready", "ready after replay") original = obs.wait_trace(trace_id, lambda s: len(by_name(s, "process attachments.process.v1")) >= 5, TRACE_WAIT) print(f"original trace {trace_id}: {len(by_name(original, 'process attachments.process.v1'))} attempts, " f"last outcome {sorted(by_name(original, 'process attachments.process.v1'), key=lambda s: s['start'])[-1]['attrs'].get('messenger.job.outcome')}") found = None deadline = time.monotonic() + TRACE_WAIT while found is None and time.monotonic() < deadline: for item in obs.tempo_search(f'{{ name = "create attachments" && span.messenger.job.id = "{job["job_id"]}" }}', minutes=15, limit=20): if item["traceID"].rjust(32, "0") != trace_id: found = item["traceID"].rjust(32, "0") if found is None: time.sleep(2) check(found, "replay trace not found") spans = obs.wait_trace(found, lambda s: by_name(s, "commit result"), TRACE_WAIT) print(obs.span_tree(spans)) create = by_name(spans, "create attachments")[0] linked = {link["trace_id"] for link in create["links"]} check(trace_id in linked, f"create span of the replay links to the original trace {trace_id}: {linked}") print(f"replay trace {found}: create span links -> {sorted(linked)} (original upload trace {trace_id})") check(by_name(spans, "ops replay"), "ops replay span is the root of the new trace") check(by_name(spans, "process attachments.process.v1")[0]["attrs"].get("messenger.job.outcome") == "ready", "replayed attempt ready") print("PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it") def test_history(): user = new_user("history") chat = new_chat(user, title="History chat") trace_id, traceparent = obs.new_traceparent() call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": "history"}, headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent}) obs.wait_trace(trace_id, seconds=TRACE_WAIT) obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"', 1, LOKI_WAIT) query = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})' stamp = round(time.time() - 30) # значение на момент t-30 с (параметр time запроса API Prometheus) rows = obs.prom_query(query, at=stamp) before = rows[0][1] if rows else None check(before, "metric sample before restart") print(f"before restart: trace {trace_id} found, its log found, {query} at t-30s = {before:g}") compose("restart", "prometheus", "loki", "tempo") deadline = time.monotonic() + 120 while True: try: spans = obs.tempo_trace(trace_id) logs = obs.loki_query(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"') rows = obs.prom_query(query, at=stamp) after = rows[0][1] if rows else None if spans and logs and after == before: break except OSError: pass check(time.monotonic() < deadline, "history not available 120 s after restart") time.sleep(3) print(f"after restart of prometheus, loki, tempo: trace {trace_id} ({len(spans)} spans), " f"{len(logs)} log record(s), metric at t-30s = {after:g} (same value)") print("PASS: persistent volumes keep traces, logs and metrics across restarts of the stores") SCENARIOS = {"request": test_request, "upload-chain": test_upload_chain, "retry-chain": test_retry_chain, "replay-link": test_replay_link, "history": test_history} if __name__ == "__main__": names = sys.argv[1:] or list(SCENARIOS) unknown = [n for n in names if n not in SCENARIOS] if unknown: raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}") run_tests([SCENARIOS[n] for n in names])