# 2026-11-14T17:00:59Z python3 student-tests/obs_correlation.py --retry --output evidence/obs-correlation.json
[   0.51s] message sent {"request_id": "f59ccc9eae9f4e118ad806399fc7cdda", "trace_id": "b2f0dd26b5b1e1d1410c2d357a69e36e", "instance": "670c04e57dee"}
[   2.60s] message trace in Tempo {"spans": 4, "server_ms": 2.281, "db_spans": ["db get_session_user", "db get_chat_for", "db add_message"]}
[   2.62s] structured API log in Loki {"event": "http.request", "trace_id": "b2f0dd26b5b1e1d1410c2d357a69e36e", "span_id": "aedaa74d918d11cb", "severity": "INFO", "service_name": "messenger-api", "line": "POST /api/v1/chats/{chat_id}/messages -> 201 2.2ms request_id=f59ccc9eae9f4e118ad806399fc7cdda"}
[   2.96s] upload accepted and ready {"attachment_id": "0020bba2-7684-49fc-ae0e-05e30e9ee64f", "trace_id": "69a4cabe0565f4477758b30a0ffe8958", "status": "ready"}
[   6.06s] one trace upload -> relay -> worker {"spans": 26, "services": ["messenger-api", "messenger-relay", "messenger-worker"], "job_id": "bb3deb41-b32a-4109-9056-ba8d47c280f6", "outbox_traceparent": "00-69a4cabe0565f4477758b30a0ffe8958-c31e4d41fac37f54-03", "process_parent_is_send": true, "send_parent_is_create": true, "worker_s3_puts": 4, "api_to_worker_gap_ms": 2.4, "process_ms": 132.022, "outcome": "ready"}
[   6.07s] worker structured logs in Loki by trace_id {"events": ["job.claimed", "job.completed"], "job_id": "bb3deb41-b32a-4109-9056-ba8d47c280f6", "span_id": "01c12371a60672d0", "line": "job bb3deb41-b32a-4109-9056-ba8d47c280f6 attempt 1 -> ready in 127ms"}
[  16.73s] retry chain in one trace {"trace_id": "c44375212d405aabc788b452a755b889", "attempts": 2, "outcomes": ["retry", "ready"], "retry_delay_s": 1.01, "error_type": "storage_unavailable"}
[  23.66s] worker restored without fault injection 
[  26.97s] no secrets/bodies in telemetry, no ID labels in metrics {"leaks": {"loki:message_text": 0, "loki:password": 0, "loki:token": 0, "loki:filename": 0, "tempo:message_text": 0, "tempo:password": 0, "tempo:token": 0, "tempo:filename": 0}, "metric_labels": ["__name__", "db_client_connection_pool_name", "db_client_connection_state", "db_operation_name", "db_system_name", "deployment_environment_name", "docker_cli_cobra_command_path", "error_type", "http_request_method", "http_response_status_class", "http_response_status_code", "http_route", "instance", "job", "le", "messenger_job_outcome", "messenger_job_status", "messenger_job_type", "messenger_s3_operation", "otel_component_name", "otel_component_type", "otel_span_parent_origin", "otel_span_sampling_result", "outcome", "server_address", "server_port", "service_version", "telemetry_sdk_language", "telemetry_sdk_name", "telemetry_sdk_version"]}
PASS: correlation request->trace->log and upload->relay->worker incl. retry
exit=0
