Files
mtusi2026lab6/student-tests/test_observability_retry.py
T

59 lines
3.0 KiB
Python

"""Цепочка трассировки с повторами (уровень 4): все попытки одного задания находятся в ОДНОЙ трассе.
python student-tests/test_observability_retry.py
Worker пересоздаётся с внедрением сбоя (fail:after_claim, 3 попытки), затем возвращается к штатной конфигурации.
"""
import json
import os
import sys
import time
import unittest
import urllib.request
import uuid
sys.path.insert(0, os.path.dirname(__file__))
from test_api import BASE, call, register # noqa: E402
from test_attachments import PNG # noqa: E402
from test_observability import loki, spans_of, tempo_trace # noqa: E402
from test_pipeline import wait_status # noqa: E402
from test_resilience5 import compose, healthy, recreate_worker, wait # noqa: E402
class RetryTraceTest(unittest.TestCase):
@classmethod
def tearDownClass(cls):
recreate_worker()
def test_all_attempts_in_one_trace_and_logs(self):
recreate_worker({"WORKER_FAULT_INJECTION": "fail:after_claim", "MAX_ATTEMPTS": "3", "RETRY_BASE_DELAY": "0.3"})
a = register("obsr")
chat = call("POST", "/api/v1/chats", {"title": "retry-trace", "member_ids": [a["user"]["id"]]}, a["token"])[2]
trace_id = uuid.uuid4().hex
req = urllib.request.Request("%s/api/v1/chats/%s/attachments?filename=r.png" % (BASE, chat["id"]), data=PNG, method="POST",
headers={"Authorization": "Bearer " + a["token"], "Content-Type": "image/png",
"traceparent": "00-%s-%s-01" % (trace_id, uuid.uuid4().hex[:16])})
with urllib.request.urlopen(req, timeout=30) as r:
att = json.loads(r.read())
failed = wait_status(a["token"], att["id"], ("failed",), 60)
self.assertEqual(failed["error_code"], "retry_exhausted")
spans, deadline = [], time.time() + 40
while time.time() < deadline:
trace = tempo_trace(trace_id)
spans = spans_of(trace) if trace else []
consumers = [s for s in spans if s["name"] == "attachment.process process"]
if len(consumers) >= 3:
break
time.sleep(2)
consumers = sorted([s for s in spans if s["name"] == "attachment.process process"], key=lambda s: int(s["attrs"]["job.attempt"]))
publishers = {s["id"]: s for s in spans if s["name"] == "attachment.process publish"}
self.assertEqual([int(c["attrs"]["job.attempt"]) for c in consumers], [1, 2, 3])
for c in consumers:
self.assertIn(c["parent"], publishers, "consumer-спан каждой попытки — потомок publish-спана той же трассы")
logs = loki('{service_name="messenger-worker"} | trace_id="%s"' % trace_id)
messages = [v[1] for _, v in logs]
self.assertEqual(messages.count("job_rescheduled"), 2)
self.assertIn("job_retry_exhausted", messages)
if __name__ == "__main__":
unittest.main()