223 lines
11 KiB
Python
223 lines
11 KiB
Python
"""OTel contract/security checks without a running DB, broker or backends.
|
|
|
|
Run in the application image, where the pinned SDK/FastAPI are installed:
|
|
docker compose run --rm --no-deps -v ./:/work -w /work api python student-tests/observability_unit.py
|
|
Only synthetic markers are used; no external exporter is configured.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json
|
|
import logging
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
# app.errors reads configuration to build response headers; no connection is made.
|
|
os.environ.setdefault("DATABASE_URL", "postgresql://unit:unit@unused/unit")
|
|
os.environ.setdefault("S3_ACCESS_KEY_ID", "synthetic-unit-key")
|
|
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "synthetic-unit-secret")
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import InMemoryLogExporter, SimpleLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import InMemoryMetricReader
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
|
from opentelemetry.trace import StatusCode
|
|
|
|
from app import telemetry
|
|
from app.trace_context import carrier, parent_context
|
|
|
|
|
|
class ObservabilityTests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
resource = Resource({"service.name": "messenger-unit"})
|
|
cls.spans = InMemorySpanExporter()
|
|
cls.traces = TracerProvider(resource=resource)
|
|
cls.traces.add_span_processor(SimpleSpanProcessor(cls.spans))
|
|
trace.set_tracer_provider(cls.traces)
|
|
cls.reader = InMemoryMetricReader()
|
|
cls.meters = MeterProvider(resource=resource, metric_readers=[cls.reader])
|
|
metrics.set_meter_provider(cls.meters)
|
|
cls.logs = InMemoryLogExporter()
|
|
cls.log_provider = LoggerProvider(resource=resource)
|
|
cls.log_provider.add_log_record_processor(SimpleLogRecordProcessor(cls.logs))
|
|
cls.logger = logging.getLogger("messenger.telemetry.events")
|
|
cls.handler = LoggingHandler(logger_provider=cls.log_provider)
|
|
cls.logger.addHandler(cls.handler)
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.logger.removeHandler(cls.handler)
|
|
cls.meters.shutdown()
|
|
cls.log_provider.shutdown()
|
|
cls.traces.shutdown()
|
|
|
|
def setUp(self):
|
|
self.spans.clear()
|
|
self.logs.clear()
|
|
|
|
def test_w3c_parent_child_and_durable_retry_preserve_trace(self):
|
|
initial = {
|
|
"traceparent": "00-12345678901234567890123456789012-1234567890123456-01",
|
|
"tracestate": "vendor=unit",
|
|
"Authorization": "Bearer MUST_NOT_PROPAGATE",
|
|
}
|
|
with telemetry.span("upload", parent=parent_context(initial)) as upload:
|
|
durable = carrier()
|
|
self.assertEqual(set(durable), {"traceparent", "tracestate"})
|
|
self.assertEqual(durable["tracestate"], "vendor=unit")
|
|
with telemetry.span("publish", parent=parent_context(durable)) as producer:
|
|
envelope = carrier()
|
|
self.assertEqual(producer.get_span_context().trace_id, upload.get_span_context().trace_id)
|
|
with telemetry.span("consume", parent=parent_context(envelope)) as consumer:
|
|
retry = carrier()
|
|
with telemetry.span("retry publish", parent=parent_context(retry)) as retried:
|
|
self.assertEqual(retried.get_span_context().trace_id, upload.get_span_context().trace_id)
|
|
by_name = {item.name: item for item in self.spans.get_finished_spans()}
|
|
self.assertEqual(by_name["publish"].parent.span_id, upload.get_span_context().span_id)
|
|
self.assertEqual(by_name["consume"].parent.span_id, producer.get_span_context().span_id)
|
|
self.assertEqual(by_name["retry publish"].parent.span_id, consumer.get_span_context().span_id)
|
|
self.assertTrue(envelope["traceparent"].endswith("-01"))
|
|
|
|
def test_unsampled_trace_flags_and_tracestate_survive(self):
|
|
initial = {
|
|
"traceparent": "00-12345678901234567890123456789012-1234567890123456-00",
|
|
"tracestate": "vendor=unit",
|
|
}
|
|
with telemetry.span("unsampled", parent=parent_context(initial)) as active:
|
|
propagated = carrier()
|
|
self.assertEqual(active.get_span_context().trace_id, int(initial["traceparent"].split("-")[1], 16))
|
|
self.assertEqual(propagated["tracestate"], "vendor=unit")
|
|
self.assertTrue(propagated["traceparent"].endswith("-00"))
|
|
self.assertEqual(self.spans.get_finished_spans(), ())
|
|
|
|
def test_missing_or_invalid_context_starts_new_root_even_inside_span(self):
|
|
with telemetry.span("unrelated maintenance") as ambient:
|
|
for candidate in (None, {}, {"traceparent": None}, {"traceparent": "broken"},
|
|
{"traceparent": "00-" + "0" * 32 + "-1234567890123456-01"}):
|
|
with telemetry.span("new root", parent=parent_context(candidate)) as active:
|
|
self.assertNotEqual(active.get_span_context().trace_id, ambient.get_span_context().trace_id)
|
|
roots = [item for item in self.spans.get_finished_spans() if item.name == "new root"]
|
|
self.assertEqual(len(roots), 5)
|
|
self.assertTrue(all(item.parent is None for item in roots))
|
|
self.assertEqual(carrier(), {})
|
|
|
|
def _request(self, *, template=None, status=202, failure=None):
|
|
secrets = {
|
|
"raw_id": "3d7b756a-cb54-4ce5-a7a8-83aa92352cee",
|
|
"filename": "PRIVATE_FILENAME.png",
|
|
"token": "PRIVATE_BEARER_TOKEN",
|
|
"body": "PRIVATE_MESSAGE_BODY",
|
|
"exception": "PRIVATE_EXCEPTION_CONNECTION_URL",
|
|
}
|
|
received = []
|
|
responses = []
|
|
scope = {
|
|
"type": "http", "asgi": {"version": "3.0"}, "http_version": "1.1",
|
|
"method": "POST", "scheme": "http", "root_path": "",
|
|
"path": f"/api/v1/chats/{secrets['raw_id']}/attachments",
|
|
"raw_path": f"/api/v1/chats/{secrets['raw_id']}/attachments".encode(),
|
|
"query_string": f"filename={secrets['filename']}".encode(),
|
|
"headers": [(b"authorization", f"Bearer {secrets['token']}".encode()),
|
|
(b"content-type", b"application/json"),
|
|
(b"x-request-id", b"unit-request-correlation")],
|
|
"client": ("127.0.0.1", 1234), "server": ("unit", 80),
|
|
}
|
|
|
|
async def receive():
|
|
received.append(True)
|
|
return {"type": "http.request", "body": secrets["body"].encode(), "more_body": False}
|
|
|
|
async def send(message):
|
|
responses.append(message)
|
|
|
|
async def application(scope, receive, send):
|
|
if template:
|
|
scope["route"] = SimpleNamespace(path=template)
|
|
# Prove the same secret body is available to the handler, but absent
|
|
# from the independently exported observations.
|
|
message = await receive()
|
|
assert message["body"] == secrets["body"].encode()
|
|
if failure:
|
|
raise RuntimeError(secrets["exception"])
|
|
await send({"type": "http.response.start", "status": status, "headers": []})
|
|
await send({"type": "http.response.body", "body": b"ok", "more_body": False})
|
|
|
|
run = telemetry.HttpTelemetryMiddleware(application)(scope, receive, send)
|
|
if failure:
|
|
with self.assertRaisesRegex(RuntimeError, secrets["exception"]):
|
|
asyncio.run(run)
|
|
else:
|
|
asyncio.run(run)
|
|
self.assertEqual(len(received), 1)
|
|
return secrets, responses
|
|
|
|
def _exported(self):
|
|
spans = self.spans.get_finished_spans()
|
|
logs = [entry.log_record for entry in self.logs.get_finished_logs()]
|
|
data = self.reader.get_metrics_data()
|
|
point_attrs = []
|
|
for resource_metrics in data.resource_metrics:
|
|
for scope_metrics in resource_metrics.scope_metrics:
|
|
for metric in scope_metrics.metrics:
|
|
for point in metric.data.data_points:
|
|
point_attrs.append(dict(point.attributes))
|
|
value = {
|
|
"spans": [{"name": item.name, "attrs": dict(item.attributes),
|
|
"events": [{"name": event.name, "attrs": dict(event.attributes)}
|
|
for event in item.events]} for item in spans],
|
|
"logs": [{"body": item.body, "attrs": dict(item.attributes or {})} for item in logs],
|
|
"metrics": point_attrs,
|
|
}
|
|
return spans, logs, point_attrs, json.dumps(value, default=str)
|
|
|
|
def test_http_route_template_and_request_trace_log_correlation_without_secrets(self):
|
|
template = "/api/v1/chats/{chat_id}/attachments"
|
|
secrets, responses = self._request(template=template)
|
|
spans, logs, metric_attrs, exported = self._exported()
|
|
self.assertEqual(len(spans), 1)
|
|
self.assertEqual(spans[0].name, "POST " + template)
|
|
self.assertTrue(metric_attrs)
|
|
self.assertTrue(all(item["route"] in {template, "unmatched"} for item in metric_attrs))
|
|
self.assertTrue(all(set(item) <= {"method", "route", "status_class"} for item in metric_attrs))
|
|
self.assertEqual(len(logs), 1)
|
|
self.assertEqual(logs[0].attributes["request_id"], "unit-request-correlation")
|
|
self.assertEqual(logs[0].trace_id, spans[0].context.trace_id)
|
|
self.assertEqual(logs[0].span_id, spans[0].context.span_id)
|
|
headers = dict(responses[0]["headers"])
|
|
self.assertEqual(headers[b"x-request-id"], b"unit-request-correlation")
|
|
self.assertEqual(headers[b"x-trace-id"].decode(), f"{spans[0].context.trace_id:032x}")
|
|
for value in secrets.values():
|
|
self.assertNotIn(value, exported)
|
|
|
|
def test_unknown_route_and_exception_message_do_not_escape_into_telemetry(self):
|
|
secrets, _ = self._request(status=404)
|
|
_, _, _, exported = self._exported()
|
|
self.assertEqual(self.spans.get_finished_spans()[0].name, "POST unmatched")
|
|
for value in secrets.values():
|
|
self.assertNotIn(value, exported)
|
|
self.spans.clear()
|
|
self.logs.clear()
|
|
secrets, _ = self._request(failure=True)
|
|
spans, logs, metric_attrs, exported = self._exported()
|
|
self.assertEqual(spans[0].status.status_code, StatusCode.ERROR)
|
|
self.assertEqual(spans[0].attributes["exception_type"], "RuntimeError")
|
|
self.assertFalse(spans[0].events, "No exception stack/message should be auto-recorded")
|
|
self.assertEqual(logs[0].attributes["status_code"], 500)
|
|
self.assertTrue(any(item["status_class"] == "5xx" for item in metric_attrs))
|
|
for value in secrets.values():
|
|
self.assertNotIn(value, exported)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main(verbosity=2)
|