forked from mtusi2026/mtusi2026lab6
259 lines
16 KiB
Python
259 lines
16 KiB
Python
"""Lab6: связь запроса, структурного лога и трассы; сквозная трасса upload -> worker, включая retry.
|
||
|
||
python student-tests/test_telemetry.py # все сценарии
|
||
python student-tests/test_telemetry.py upload-chain # один (список SCENARIOS ниже)
|
||
|
||
Нужен запущенный стенд (compose.yaml, FAULT_INJECTION=1 для retry/replay). Клиент сам задаёт W3C
|
||
traceparent запроса, поэтому трасса находится в Tempo по известному trace_id, без поиска «похожей».
|
||
Тест печатает дерево спанов (без данных пользователя) — это доказательство для отчёта.
|
||
|
||
Сценарии:
|
||
request уровень 3: POST сообщения -> server-спан API с дочерними спанами PostgreSQL/Redis,
|
||
запись лога event=http.request с тем же trace_id/span_id и request_id из X-Request-Id;
|
||
текст сообщения, пароль и токен не попали ни в логи (Loki), ни в атрибуты спанов.
|
||
upload-chain уровень 4: одна трасса API(server) -> create(producer, outbox) -> dispatcher publish
|
||
(producer) -> worker process (consumer) -> S3 PutObject x4 и COMMIT ready; логи worker
|
||
с тем же trace_id.
|
||
retry-chain уровень 4: временная ошибка S3 в worker (fault worker_s3_error) -> в той же трассе
|
||
попытка 1 (ERROR, outcome=retry) -> новая публикация -> попытка 2 (ready).
|
||
replay-link уровень 4: попытки исчерпаны -> failed -> ops.py replay создаёт НОВУЮ трассу со
|
||
ссылкой (link) на исходную загрузку, worker завершает её в ready.
|
||
history уровень 3: после restart prometheus/loki/tempo (volumes сохранены) прежние трасса,
|
||
логи и метрика по-прежнему находятся.
|
||
"""
|
||
import json
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import uuid
|
||
|
||
import obs
|
||
from common import ROOT, call, check, compose, new_chat, new_user, run_tests, wait_attachment
|
||
from images import png
|
||
|
||
LOKI_WAIT = 60
|
||
TRACE_WAIT = 60
|
||
|
||
|
||
def upload_traced(user, chat_id, data, traceparent, filename="trace.png"):
|
||
reply = call("POST", f"/api/v1/chats/{chat_id}/attachments?filename={filename}", 202, raw=data,
|
||
token=user["token"], headers={"Content-Type": "image/png", "traceparent": traceparent})
|
||
return reply
|
||
|
||
|
||
def by_name(spans, name, service=None):
|
||
return [s for s in spans if s["name"] == name and (service is None or s["service"] == service)]
|
||
|
||
|
||
def parent_of(spans, span):
|
||
return next((s for s in spans if s["span_id"] == span["parent_id"]), None)
|
||
|
||
|
||
def fault(point, count=1):
|
||
for n in range(count):
|
||
subprocess.run(["docker", "compose", "exec", "-T", "worker", "sh", "-c", f"touch /faults/{point}.{n + 1}"],
|
||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||
|
||
|
||
def ops(*args):
|
||
result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)],
|
||
cwd=ROOT, capture_output=True, check=True)
|
||
return json.loads(result.stdout.decode("utf-8"))
|
||
|
||
|
||
# ---------- сценарии ----------
|
||
|
||
def test_request():
|
||
user = new_user("trace")
|
||
chat = new_chat(user, title="Trace chat")
|
||
marker = f"secret-text-{uuid.uuid4().hex}"
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": marker},
|
||
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent})
|
||
request_id = reply.headers["x-request-id"]
|
||
check(reply.headers.get("x-trace-id") == trace_id, "X-Trace-Id must continue the client's traceparent")
|
||
print(f"POST message: 201, X-Request-Id={request_id}, trace_id={trace_id} (client traceparent)")
|
||
|
||
spans = obs.wait_trace(trace_id, lambda s: by_name(s, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")
|
||
and any(x["kind"] == "CLIENT" and x["name"] == "INSERT" for x in s), TRACE_WAIT)
|
||
server = by_name(spans, "POST /api/v1/chats/{chat_id}/messages", "messenger-api")[0]
|
||
check(server["kind"] == "SERVER" and server["attrs"].get("http.response.status_code") == 201, "server span")
|
||
check(server["attrs"].get("http.route") == "/api/v1/chats/{chat_id}/messages", "route template in span")
|
||
db = [s for s in spans if s["parent_id"] == server["span_id"] and "db.statement" in s["attrs"]
|
||
or "db.query.text" in s["attrs"]]
|
||
check(db, "PostgreSQL client spans under the server span")
|
||
check(by_name(spans, "redis GET", "messenger-api"), "Redis session lookup span")
|
||
print(obs.span_tree(spans))
|
||
|
||
rows = obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}" | event="http.request"',
|
||
1, LOKI_WAIT)
|
||
labels, line = rows[0]
|
||
check(labels.get("request_id") == request_id, "log request_id == X-Request-Id")
|
||
check(labels.get("span_id") == server["span_id"], "log span_id == server span id")
|
||
for field in ("service_name", "severity_text", "event", "trace_id", "span_id"):
|
||
check(labels.get(field), f"log field {field} is missing")
|
||
print(f"Loki: {line!r} event={labels['event']} severity={labels['severity_text']} "
|
||
f"service={labels['service_name']} trace_id={labels['trace_id']} span_id={labels['span_id']} "
|
||
f"request_id={labels['request_id']}")
|
||
|
||
# Ни текст сообщения, ни пароль, ни токен не попали в телеметрию.
|
||
for secret, what in ((marker, "message text"), (user["password"], "password"), (user["token"][:24], "token")):
|
||
found = obs.loki_query(f'{{service_name=~"messenger-.*"}} |= `{secret}`', minutes=10, limit=5)
|
||
check(not found, f"{what} found in Loki logs")
|
||
check(not any(secret in str(v) for s in spans for v in s["attrs"].values()), f"{what} found in span attributes")
|
||
check(not any(k.lower().startswith(("http.request.header.authorization", "enduser")) for s in spans
|
||
for k in s["attrs"]), "Authorization/enduser attributes in spans")
|
||
print("PASS: request -> server span with DB/Redis children -> structured log with the same trace_id/span_id "
|
||
"and request_id; no message text, password or token in logs/span attributes")
|
||
|
||
|
||
def test_upload_chain():
|
||
user = new_user("chain")
|
||
chat = new_chat(user, title="Chain chat")
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
accepted = upload_traced(user, chat["id"], png(320, 200), traceparent).json
|
||
ready = wait_attachment(user, accepted["id"])
|
||
check(ready["status"] == "ready", f"attachment is {ready['status']}")
|
||
|
||
def complete(spans):
|
||
return by_name(spans, "commit result", "messenger-worker") and by_name(spans, "cleanup staging")
|
||
|
||
spans = obs.wait_trace(trace_id, complete, TRACE_WAIT)
|
||
print(obs.span_tree(spans))
|
||
server = by_name(spans, "POST /api/v1/chats/{chat_id}/attachments", "messenger-api")[0]
|
||
create = by_name(spans, "create attachments", "messenger-api")[0]
|
||
publish = by_name(spans, "publish attachments", "messenger-dispatcher")[0]
|
||
consume = by_name(spans, "process attachments.process.v1", "messenger-worker")[0]
|
||
check(create["kind"] == "PRODUCER" and publish["kind"] == "PRODUCER" and consume["kind"] == "CONSUMER", "kinds")
|
||
check(parent_of(spans, create)["span_id"] == server["span_id"], "create is a child of the API request")
|
||
check(publish["parent_id"] == create["span_id"], "publish (dispatcher) is a child of create (API outbox)")
|
||
check(consume["parent_id"] == publish["span_id"], "consume (worker) is a child of publish (W3C via AMQP headers)")
|
||
check(consume["attrs"].get("messenger.job.outcome") == "ready", "consumer outcome ready")
|
||
puts = [s for s in by_name(spans, "S3 PutObject", "messenger-worker")
|
||
if s["attrs"].get("aws.s3.key", "").startswith(f"attachments/{accepted['id']}/v1/")]
|
||
check(len(puts) == 4, f"4 PUTs of the final objects, got {len(puts)}")
|
||
check({s["service"] for s in spans} >= {"messenger-api", "messenger-dispatcher", "messenger-worker"}, "services")
|
||
events = {labels["event"] for labels, _ in obs.wait_logs(
|
||
f'{{service_name="messenger-worker"}} | trace_id="{trace_id}"', 2, LOKI_WAIT)}
|
||
check({"job.claimed", "job.ready"} <= events, f"worker logs with this trace_id: {events}")
|
||
print(f"worker logs with trace_id {trace_id}: {sorted(events)}")
|
||
print(f"PASS: one trace {trace_id}: API 202 -> outbox create -> dispatcher publish -> worker consume -> "
|
||
f"4 S3 objects + COMMIT ready ({len(spans)} spans, services: "
|
||
f"{', '.join(sorted({s['service'] for s in spans}))})")
|
||
|
||
|
||
def test_retry_chain():
|
||
user = new_user("retry")
|
||
chat = new_chat(user, title="Retry chat")
|
||
before = obs.prom_value('sum(messenger_job_duration_seconds_count{outcome="retry"})', 0.0)
|
||
fault("worker_s3_error", 1)
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
accepted = upload_traced(user, chat["id"], png(64, 48), traceparent).json
|
||
ready = wait_attachment(user, accepted["id"])
|
||
check(ready["status"] == "ready", f"attachment is {ready['status']}")
|
||
|
||
def complete(spans):
|
||
return len(by_name(spans, "process attachments.process.v1")) >= 2 and by_name(spans, "commit result")
|
||
|
||
spans = obs.wait_trace(trace_id, complete, TRACE_WAIT)
|
||
print(obs.span_tree(spans))
|
||
attempts = sorted(by_name(spans, "process attachments.process.v1", "messenger-worker"), key=lambda s: s["start"])
|
||
first, second = attempts[0], attempts[-1]
|
||
check(first["status"] == "ERROR" and first["attrs"].get("messenger.job.outcome") == "retry", "attempt 1 = retry")
|
||
check(first["attrs"].get("error.type") == "storage_unavailable", "attempt 1 error.type")
|
||
check(second["attrs"].get("messenger.job.outcome") == "ready", "attempt 2 = ready")
|
||
retry_create = [s for s in by_name(spans, "create attachments") if s["parent_id"] == first["span_id"]]
|
||
check(retry_create, "retry message is created inside attempt 1 (outbox in the same transaction)")
|
||
second_publish = parent_of(spans, second)
|
||
check(second_publish["name"] == "publish attachments" and second_publish["parent_id"] == retry_create[0]["span_id"],
|
||
"attempt 2 consumes the retry publication")
|
||
check(len(by_name(spans, "publish attachments")) == 2, "two publications in one trace")
|
||
rows = obs.wait_logs(f'{{service_name="messenger-worker"}} | trace_id="{trace_id}" | event="job.retry"', 1, LOKI_WAIT)
|
||
print(f"worker log: {rows[0][1]!r} error_type={rows[0][0].get('error_type')}")
|
||
after = obs.wait_metric('sum(messenger_job_duration_seconds_count{outcome="retry"})', lambda v: v > before, 40)
|
||
print(f"messenger_job_duration_seconds_count{{outcome=\"retry\"}}: {before:g} -> {after:g}")
|
||
print(f"PASS: retry in one trace {trace_id}: attempt 1 ERROR storage_unavailable -> create -> publish -> "
|
||
f"attempt 2 ready; retry counted in metrics and logged")
|
||
|
||
|
||
def test_replay_link():
|
||
user = new_user("replay")
|
||
chat = new_chat(user, title="Replay chat")
|
||
fault("worker_s3_error", 5)
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
accepted = upload_traced(user, chat["id"], png(32, 32), traceparent).json
|
||
failed = wait_attachment(user, accepted["id"], statuses=("failed",), seconds=120)
|
||
check(failed["error_code"] == "retry_exhausted", f"error_code {failed['error_code']}")
|
||
job = next(j for j in ops("failed", "--limit", "20") if j["attachment_id"] == accepted["id"])
|
||
result = ops("replay", job["job_id"], "--reason", "lab6 link test")
|
||
check("replayed" in result, f"replay refused: {result}")
|
||
ready = wait_attachment(user, accepted["id"])
|
||
check(ready["status"] == "ready", "ready after replay")
|
||
original = obs.wait_trace(trace_id, lambda s: len(by_name(s, "process attachments.process.v1")) >= 5, TRACE_WAIT)
|
||
print(f"original trace {trace_id}: {len(by_name(original, 'process attachments.process.v1'))} attempts, "
|
||
f"last outcome {sorted(by_name(original, 'process attachments.process.v1'), key=lambda s: s['start'])[-1]['attrs'].get('messenger.job.outcome')}")
|
||
found = None
|
||
deadline = time.monotonic() + TRACE_WAIT
|
||
while found is None and time.monotonic() < deadline:
|
||
for item in obs.tempo_search(f'{{ name = "create attachments" && span.messenger.job.id = "{job["job_id"]}" }}',
|
||
minutes=15, limit=20):
|
||
if item["traceID"].rjust(32, "0") != trace_id:
|
||
found = item["traceID"].rjust(32, "0")
|
||
if found is None:
|
||
time.sleep(2)
|
||
check(found, "replay trace not found")
|
||
spans = obs.wait_trace(found, lambda s: by_name(s, "commit result"), TRACE_WAIT)
|
||
print(obs.span_tree(spans))
|
||
create = by_name(spans, "create attachments")[0]
|
||
linked = {link["trace_id"] for link in create["links"]}
|
||
check(trace_id in linked, f"create span of the replay links to the original trace {trace_id}: {linked}")
|
||
print(f"replay trace {found}: create span links -> {sorted(linked)} (original upload trace {trace_id})")
|
||
check(by_name(spans, "ops replay"), "ops replay span is the root of the new trace")
|
||
check(by_name(spans, "process attachments.process.v1")[0]["attrs"].get("messenger.job.outcome") == "ready",
|
||
"replayed attempt ready")
|
||
print("PASS: replay = new trace (root 'ops replay') with a link to the original upload; worker finished it")
|
||
|
||
|
||
def test_history():
|
||
user = new_user("history")
|
||
chat = new_chat(user, title="History chat")
|
||
trace_id, traceparent = obs.new_traceparent()
|
||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=user["token"], body={"text": "history"},
|
||
headers={"Idempotency-Key": uuid.uuid4().hex, "traceparent": traceparent})
|
||
obs.wait_trace(trace_id, seconds=TRACE_WAIT)
|
||
obs.wait_logs(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"', 1, LOKI_WAIT)
|
||
query = 'sum(http_server_request_duration_seconds_count{job="messenger-api"})'
|
||
stamp = round(time.time() - 30) # значение на момент t-30 с (параметр time запроса API Prometheus)
|
||
rows = obs.prom_query(query, at=stamp)
|
||
before = rows[0][1] if rows else None
|
||
check(before, "metric sample before restart")
|
||
print(f"before restart: trace {trace_id} found, its log found, {query} at t-30s = {before:g}")
|
||
compose("restart", "prometheus", "loki", "tempo")
|
||
deadline = time.monotonic() + 120
|
||
while True:
|
||
try:
|
||
spans = obs.tempo_trace(trace_id)
|
||
logs = obs.loki_query(f'{{service_name="messenger-api"}} | trace_id="{trace_id}"')
|
||
rows = obs.prom_query(query, at=stamp)
|
||
after = rows[0][1] if rows else None
|
||
if spans and logs and after == before:
|
||
break
|
||
except OSError:
|
||
pass
|
||
check(time.monotonic() < deadline, "history not available 120 s after restart")
|
||
time.sleep(3)
|
||
print(f"after restart of prometheus, loki, tempo: trace {trace_id} ({len(spans)} spans), "
|
||
f"{len(logs)} log record(s), metric at t-30s = {after:g} (same value)")
|
||
print("PASS: persistent volumes keep traces, logs and metrics across restarts of the stores")
|
||
|
||
|
||
SCENARIOS = {"request": test_request, "upload-chain": test_upload_chain, "retry-chain": test_retry_chain,
|
||
"replay-link": test_replay_link, "history": test_history}
|
||
|
||
if __name__ == "__main__":
|
||
names = sys.argv[1:] or list(SCENARIOS)
|
||
unknown = [n for n in names if n not in SCENARIOS]
|
||
if unknown:
|
||
raise SystemExit(f"unknown scenarios {unknown}; available: {', '.join(SCENARIOS)}")
|
||
run_tests([SCENARIOS[n] for n in names])
|