95 lines
4.3 KiB
Python
95 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""The durable job carrier preserves one trace across broker boundaries."""
|
|
|
|
import json
|
|
import sys
|
|
import unittest
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from unittest.mock import Mock, patch
|
|
from uuid import uuid4
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from opentelemetry import trace # noqa: E402
|
|
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
|
|
from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402
|
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402
|
|
from opentelemetry.trace import SpanKind # noqa: E402
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator # noqa: E402
|
|
|
|
|
|
exporter = InMemorySpanExporter()
|
|
provider = TracerProvider()
|
|
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
|
trace.set_tracer_provider(provider)
|
|
|
|
from src.pipeline_jobs import job_payload # noqa: E402
|
|
from src import worker # noqa: E402
|
|
|
|
|
|
class TelemetryTest(unittest.TestCase):
|
|
def setUp(self):
|
|
exporter.clear()
|
|
|
|
def test_failed_job_has_low_cardinality_metric_and_error_span(self):
|
|
item = {
|
|
"job_id": str(uuid4()), "attachment_id": str(uuid4()),
|
|
"type": "attachment.process.v1", "attempt": 1,
|
|
}
|
|
counter, duration = Mock(), Mock()
|
|
with patch.object(worker, "process_message", return_value="failed"), \
|
|
patch.object(worker, "job_instruments", return_value=(counter, duration)):
|
|
self.assertEqual(worker.traced_process_message(item), "failed")
|
|
labels = {"job_type": "attachment.process.v1", "outcome": "failed"}
|
|
counter.add.assert_called_once_with(1, labels)
|
|
self.assertEqual(duration.record.call_args.args[1], labels)
|
|
self.assertEqual(exporter.get_finished_spans()[-1].status.status_code.name, "ERROR")
|
|
|
|
def test_upload_publish_consume_and_retry_share_causal_trace(self):
|
|
propagator = TraceContextTextMapPropagator()
|
|
tracer = trace.get_tracer(__name__)
|
|
row = {
|
|
"job_id": uuid4(), "attachment_id": uuid4(), "pipeline_version": 1,
|
|
"attempt": 1, "created_at": datetime.now(timezone.utc),
|
|
}
|
|
|
|
with tracer.start_as_current_span("upload") as upload:
|
|
upload_carrier = {}
|
|
propagator.inject(upload_carrier)
|
|
row.update(upload_carrier)
|
|
|
|
producer_parent = propagator.extract(upload_carrier)
|
|
with tracer.start_as_current_span("publish", context=producer_parent, kind=SpanKind.PRODUCER) as producer:
|
|
producer_carrier = {}
|
|
propagator.inject(producer_carrier)
|
|
message = {**job_payload(row), **producer_carrier}
|
|
|
|
with patch.object(worker, "process_message", return_value="queued"):
|
|
self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(message).encode())), "queued")
|
|
# A managed retry uses the same durable upload parent but a new
|
|
# publisher attempt; both worker attempts remain in one trace.
|
|
row["attempt"] = 2
|
|
retry_parent = propagator.extract(upload_carrier)
|
|
with tracer.start_as_current_span("retry_publish", context=retry_parent,
|
|
kind=SpanKind.PRODUCER) as retry_producer:
|
|
retry_carrier = {}
|
|
propagator.inject(retry_carrier)
|
|
retry_message = {**job_payload(row), **retry_carrier}
|
|
self.assertEqual(worker.traced_process_message(worker.parse_message(json.dumps(retry_message).encode())), "queued")
|
|
|
|
spans = exporter.get_finished_spans()
|
|
consumers = [span for span in spans if span.name == "attachments.process"]
|
|
self.assertEqual(len(consumers), 2)
|
|
self.assertEqual(consumers[0].parent.span_id, producer.get_span_context().span_id)
|
|
self.assertEqual(consumers[1].parent.span_id, retry_producer.get_span_context().span_id)
|
|
self.assertEqual(producer.parent.span_id, upload.get_span_context().span_id)
|
|
self.assertEqual(retry_producer.parent.span_id, upload.get_span_context().span_id)
|
|
self.assertEqual({span.context.trace_id for span in spans}, {upload.get_span_context().trace_id})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|