Files
mtusi2026lab6/student-tests/test_telemetry.py

259 lines
16 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Lab6: связь запроса, структурного лога и трассы; сквозная трасса upload -> worker, включая retry.
python student-tests/test_telemetry.py # все сценарии
python student-tests/test_telemetry.py upload-chain # один (список SCENARIOS ниже)
Нужен запущенный стенд (compose.yaml, FAULT_INJECTION=1 для retry/replay). Клиент сам задаёт W3C
traceparent запроса, поэтому трасса находится в Tempo по известному trace_id, без поиска «похожей».
Тест печатает дерево спанов (без данных пользователя) — это доказательство для отчёта.
Сценарии:
request уровень 3: POST сообщения -> server-спан API с дочерними спанами PostgreSQL/Redis,
запись лога event=http.request с тем же trace_id/span_id и request_id из X-Request-Id;
текст сообщения, пароль и токен не попали ни в логи (Loki), ни в атрибуты спанов.
upload-chain уровень 4: одна трасса API(server) -> create(producer, outbox) -> dispatcher publish
(producer) -> worker process (consumer) -> S3 PutObject x4 и COMMIT ready; логи worker
с тем же trace_id.
retry-chain уровень 4: временная ошибка S3 в worker (fault worker_s3_error) -> в той же трассе
попытка 1 (ERROR, outcome=retry) -> новая публикация -> попытка 2 (ready).
replay-link уровень 4: попытки исчерпаны -> failed -> ops.py replay создаёт НОВУЮ трассу со
ссылкой (link) на исходную загрузку, worker завершает её в ready.
history уровень 3: после restart prometheus/loki/tempo (volumes сохранены) прежние трасса,
логи и метрика по-прежнему находятся.
"""
import json
import subprocess
import sys
import time
import uuid
import obs
from common import ROOT, call, check, compose, new_chat, new_user, run_tests, wait_attachment
from images import png
LOKI_WAIT = 60
TRACE_WAIT = 60
def upload_traced(user, chat_id, data, traceparent, filename="trace.png"):
reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", 202, raw=data,
token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent})
return reply
def by_name(spans, name, service=None):
return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)]
def parent_of(spans, span):
return next((s for s in spans if s["span_id"] == span["parent_id"]), None)
def fault(point, count=1):
for n in range(count):
subprocess.run(["docker", "compose", "exec", "-T", "worker", "sh", "-c", f"touch /faults/{point}.{n + 1}"],
cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def ops(*args):
result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)],
cwd=ROOT, capture_output=True, check=True)
return json.loads(result.stdout.decode("utf-8"))
# ---------- сценарии ----------
def test_request():
user = new_user("trace")
chat = new_chat(user, title="Trace chat")
marker = f"secret-text-{uuid.uuid4().hex}"
trace_id, traceparent = obs.new_traceparent()
reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": marker},
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent})
request_id = reply.headers["x-request-id"]
check(reply.headers.get("x-trace-id") == trace_id, "X-Trace-Id must continue the client's traceparent")
print(f"POST message: 201, X-Request-Id={request_id}, trace_id={trace_id} (client traceparent)")
spans = obs.wait_trace(trace_id, lambda s: by_name(s, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")
and any(x["kind"] == "CLIENT" and x["name"] == "INSERT" for x in s), TRACE_WAIT)
server = by_name(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")[0]
check(server["kind"] == "SERVER" and server["attrs"].get("http.response.status_code") == 201, "server span")
check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span")
db = [s for s in spans if s["parent_id"] == server["span_id"] and "db.statement" in s["attrs"]
or "db.query.text" in s["attrs"]]
check(db, "PostgreSQL client spans under the server span")
check(by_name(spans, "redis GET", "messenger-api"), "Redis session lookup span")
print(obs.span_tree(spans))
rows = obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}" | event="http.request"',
1, LOKI_WAIT)
labels, line = rows[0]
check(labels.get("request_id") == request_id, "log request_id == X-Request-Id")
check(labels.get("span_id") == server["span_id"], "log span_id == server span id")
for field in ("service_name", "severity_text", "event", "trace_id", "span_id"):
check(labels.get(field), f"log field {field} is missing")
print(f"Loki: {line!r} event={labels['event']} severity={labels['severity_text']} "
f"service={labels['service_name']} trace_id={labels['trace_id']} span_id={labels['span_id']} "
f"request_id={labels['request_id']}")
# Ни текст сообщения, ни пароль, ни токен не попали в телеметрию.
for secret, what in ((marker, "message text"), (user["password"], "password"), (user["token"][:24], "token")):
found = obs.loki_query(f'{{service_name=~"messenger-.*"}} |= `{secret}`', minutes=10, limit=5)
check(not found, f"{what} found in Loki logs")
check(not any(secret in str(v) for s in spans for v in s["attrs"].values()), f"{what} found in span attributes")
check(not any(k.lower().startswith(("http.request.header.authorization", "enduser")) for s in spans
for k in s["attrs"]), "Authorization/enduser attributes in spans")
print("PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id "
"and request_id; no message text, password or token in logs/span attributes")
def test_upload_chain():
user = new_user("chain")
chat = new_chat(user, title="Chain chat")
trace_id, traceparent = obs.new_traceparent()
accepted = upload_traced(user, chat["id"], png(320, 200), traceparent).json
ready = wait_attachment(user, accepted["id"])
check(ready["status"] == "ready", f"attachment is {ready['status']}")
def complete(spans):
return by_name(spans, "commit result", "messenger-worker") and by_name(spans, "cleanup staging")
spans = obs.wait_trace(trace_id, complete, TRACE_WAIT)
print(obs.span_tree(spans))
server = by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api")[0]
create = by_name(spans, "create attachments", "messenger-api")[0]
publish = by_name(spans, "publish attachments", "messenger-dispatcher")[0]
consume = by_name(spans, "process attachments.process.v1", "messenger-worker")[0]
check(create["kind"] == "PRODUCER" and publish["kind"] == "PRODUCER" and consume["kind"] == "CONSUMER", "kinds")
check(parent_of(spans, create)["span_id"] == server["span_id"], "create is a child of the API request")
check(publish["parent_id"] == create["span_id"], "publish (dispatcher) is a child of create (API outbox)")
check(consume["parent_id"] == publish["span_id"], "consume (worker) is a child of publish (W3C via AMQP headers)")
check(consume["attrs"].get("messenger.job.outcome") == "ready", "consumer outcome ready")
puts = [s for s in by_name(spans, "S3 PutObject", "messenger-worker")
if s["attrs"].get("aws.s3.key", "").startswith(f"attachments/{accepted['id']}/v1/")]
check(len(puts) == 4, f"4 PUTs of the final objects, got {len(puts)}")
check({s["service"] for s in spans} >= {"messenger-api", "messenger-dispatcher", "messenger-worker"}, "services")
events = {labels["event"] for labels, _ in obs.wait_logs(
f'{{service_name="messenger-worker"}} | trace_id="{trace_id}"', 2, LOKI_WAIT)}
check({"job.claimed", "job.ready"} <= events, f"worker logs with this trace_id: {events}")
print(f"worker logs with trace_id {trace_id}: {sorted(events)}")
print(f"PASS: one trace {trace_id}: API 202 -> outbox create -> dispatcher publish -> worker consume -> "
f"4 S3 objects + COMMIT ready ({len(spans)} spans, services: "
f"{', '.join(sorted({s['service'] for s in spans}))})")
def test_retry_chain():
user = new_user("retry")
chat = new_chat(user, title="Retry chat")
before = obs.prom_value('sum(messenger_job_duration_seconds_count{outcome="retry"})', 0.0)
fault("worker_s3_error", 1)
trace_id, traceparent = obs.new_traceparent()
accepted = upload_traced(user, chat["id"], png(64, 48), traceparent).json
ready = wait_attachment(user, accepted["id"])
check(ready["status"] == "ready", f"attachment is {ready['status']}")
def complete(spans):
return len(by_name(spans, "process attachments.process.v1")) >= 2 and by_name(spans, "commit result")
spans = obs.wait_trace(trace_id, complete, TRACE_WAIT)
print(obs.span_tree(spans))
attempts = sorted(by_name(spans, "process attachments.process.v1", "messenger-worker"), key=lambda s: s["start"])
first, second = attempts[0], attempts[-1]
check(first["status"] == "ERROR" and first["attrs"].get("messenger.job.outcome") == "retry", "attempt 1 = retry")
check(first["attrs"].get("error.type") == "storage_unavailable", "attempt 1 error.type")
check(second["attrs"].get("messenger.job.outcome") == "ready", "attempt 2 = ready")
retry_create = [s for s in by_name(spans, "create attachments") if s["parent_id"] == first["span_id"]]
check(retry_create, "retry message is created inside attempt 1 (outbox in the same transaction)")
second_publish = parent_of(spans, second)
check(second_publish["name"] == "publish attachments" and second_publish["parent_id"] == retry_create[0]["span_id"],
"attempt 2 consumes the retry publication")
check(len(by_name(spans, "publish attachments")) == 2, "two publications in one trace")
rows = obs.wait_logs(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="job.retry"', 1, LOKI_WAIT)
print(f"worker log: {rows[0][1]!r} error_type={rows[0][0].get('error_type')}")
after = obs.wait_metric('sum(messenger_job_duration_seconds_count{outcome="retry"})', lambda v: v > before, 40)
print(f"messenger_job_duration_seconds_count{{outcome=\"retry\"}}: {before:g} -> {after:g}")
print(f"PASS: retry in one trace {trace_id}: attempt 1 ERROR storage_unavailable -> create -> publish -> "
f"attempt 2 ready; retry counted in metrics and logged")
def test_replay_link():
user = new_user("replay")
chat = new_chat(user, title="Replay chat")
fault("worker_s3_error", 5)
trace_id, traceparent = obs.new_traceparent()
accepted = upload_traced(user, chat["id"], png(32, 32), traceparent).json
failed = wait_attachment(user, accepted["id"], statuses=("failed",), seconds=120)
check(failed["error_code"] == "retry_exhausted", f"error_code {failed['error_code']}")
job = next(j for j in ops("failed", "--limit", "20") if j["attachment_id"] == accepted["id"])
result = ops("replay", job["job_id"], "--reason", "lab6 link test")
check("replayed" in result, f"replay refused: {result}")
ready = wait_attachment(user, accepted["id"])
check(ready["status"] == "ready", "ready after replay")
original = obs.wait_trace(trace_id, lambda s: len(by_name(s, "process attachments.process.v1")) >= 5, TRACE_WAIT)
print(f"original trace {trace_id}: {len(by_name(original, 'process attachments.process.v1'))} attempts, "
f"last outcome {sorted(by_name(original, 'process attachments.process.v1'), key=lambda s: s['start'])[-1]['attrs'].get('messenger.job.outcome')}")
found = None
deadline = time.monotonic() + TRACE_WAIT
while found is None and time.monotonic() < deadline:
for item in obs.tempo_search(f'{{ name = "create attachments" && span.messenger.job.id = "{job["job_id"]}" }}',
minutes=15, limit=20):
if item["traceID"].rjust(32, "0") != trace_id:
found = item["traceID"].rjust(32, "0")
if found is None:
time.sleep(2)
check(found, "replay trace not found")
spans = obs.wait_trace(found, lambda s: by_name(s, "commit result"), TRACE_WAIT)
print(obs.span_tree(spans))
create = by_name(spans, "create attachments")[0]
linked = {link["trace_id"] for link in create["links"]}
check(trace_id in linked, f"create span of the replay links to the original trace {trace_id}: {linked}")
print(f"replay trace {found}: create span links -> {sorted(linked)} (original upload trace {trace_id})")
check(by_name(spans, "ops replay"), "ops replay span is the root of the new trace")
check(by_name(spans, "process attachments.process.v1")[0]["attrs"].get("messenger.job.outcome") == "ready",
"replayed attempt ready")
print("PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it")
def test_history():
user = new_user("history")
chat = new_chat(user, title="History chat")
trace_id, traceparent = obs.new_traceparent()
call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": "history"},
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent})
obs.wait_trace(trace_id, seconds=TRACE_WAIT)
obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"', 1, LOKI_WAIT)
query = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
stamp = round(time.time() - 30) # значение на момент t-30 с (параметр time запроса API Prometheus)
rows = obs.prom_query(query, at=stamp)
before = rows[0][1] if rows else None
check(before, "metric sample before restart")
print(f"before restart: trace {trace_id} found, its log found, {query} at t-30s = {before:g}")
compose("restart", "prometheus", "loki", "tempo")
deadline = time.monotonic() + 120
while True:
try:
spans = obs.tempo_trace(trace_id)
logs = obs.loki_query(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"')
rows = obs.prom_query(query, at=stamp)
after = rows[0][1] if rows else None
if spans and logs and after == before:
break
except OSError:
pass
check(time.monotonic() < deadline, "history not available 120 s after restart")
time.sleep(3)
print(f"after restart of prometheus, loki, tempo: trace {trace_id} ({len(spans)} spans), "
f"{len(logs)} log record(s), metric at t-30s = {after:g} (same value)")
print("PASS: persistent volumes keep traces, logs and metrics across restarts of the stores")
SCENARIOS = {"request": test_request, "upload-chain": test_upload_chain, "retry-chain": test_retry_chain,
"replay-link": test_replay_link, "history": test_history}
if __name__ == "__main__":
names = sys.argv[1:] or list(SCENARIOS)
unknown = [n for n in names if n not in SCENARIOS]
if unknown:
raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}")
run_tests([SCENARIOS[n] for n in names])