Files
mtusi2026lab6/app/core/worker_telemetry.py

240 lines
6.4 KiB
Python

import logging
import os
import socket
import time
from opentelemetry import metrics, trace
from opentelemetry._logs import set_logger_provider
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
OTLPMetricExporter,
)
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
from opentelemetry.instrumentation.logging import LoggingInstrumentor
from opentelemetry.propagate import extract
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.trace import SpanKind, Status, StatusCode
logger = logging.getLogger(__name__)
_initialized = False
tracer = None
jobs_processed = None
jobs_failed = None
job_duration = None
def setup_worker_telemetry():
"""Initialize OpenTelemetry for the background worker."""
global _initialized
global tracer, jobs_processed, jobs_failed, job_duration
if _initialized:
return
endpoint = os.getenv(
"OTEL_EXPORTER_OTLP_ENDPOINT",
"http://otel-collector:4318",
).rstrip("/")
service_name = os.getenv(
"OTEL_SERVICE_NAME",
"messenger-worker",
)
resource = Resource.create(
{
"service.name": service_name,
"service.version": "1.4.0",
"service.instance.id": (
os.getenv("INSTANCE_ID")
or socket.gethostname()
),
"deployment.environment.name": os.getenv(
"APP_ENV",
"lab",
),
}
)
# Distributed tracing.
tracer_provider = TracerProvider(resource=resource)
tracer_provider.add_span_processor(
BatchSpanProcessor(
OTLPSpanExporter(
endpoint=f"{endpoint}/v1/traces",
)
)
)
trace.set_tracer_provider(tracer_provider)
# Metrics.
metric_reader = PeriodicExportingMetricReader(
OTLPMetricExporter(
endpoint=f"{endpoint}/v1/metrics",
),
export_interval_millis=10000,
)
meter_provider = MeterProvider(
resource=resource,
metric_readers=[metric_reader],
)
metrics.set_meter_provider(meter_provider)
tracer = trace.get_tracer("messenger.worker")
meter = metrics.get_meter("messenger.worker")
jobs_processed = meter.create_counter(
name="jobs_processed",
description="Number of successfully processed background jobs",
unit="1",
)
jobs_failed = meter.create_counter(
name="jobs_failed",
description="Number of failed background jobs",
unit="1",
)
job_duration = meter.create_histogram(
name="job_processing_duration",
description="Background job processing duration",
unit="s",
)
# Logs.
logger_provider = LoggerProvider(resource=resource)
logger_provider.add_log_record_processor(
BatchLogRecordProcessor(
OTLPLogExporter(
endpoint=f"{endpoint}/v1/logs",
)
)
)
set_logger_provider(logger_provider)
# Add trace_id and span_id to Python log records.
LoggingInstrumentor().instrument(
tracer_provider=tracer_provider,
set_logging_format=False,
)
# Export Python logging records through OTLP.
logging_handler = LoggingHandler(
level=logging.INFO,
logger_provider=logger_provider,
)
logging.getLogger().addHandler(logging_handler)
_initialized = True
logger.info(
"OpenTelemetry initialized for worker service %s",
service_name,
)
def process_job_with_telemetry(process_function, payload):
"""Run one job inside a consumer span and record its metrics."""
if not _initialized:
setup_worker_telemetry()
started = time.perf_counter()
outcome = "other"
# Continue the trace started by the API request.
context = extract(payload.get("trace_context", {}))
with tracer.start_as_current_span(
"worker.process_job",
context=context,
kind=SpanKind.CONSUMER,
attributes={
"messaging.system": "rabbitmq",
"messaging.operation": "process",
"job.type": "image",
},
) as span:
try:
result = process_function(payload)
state = result.get("state", "unknown")
span.set_attribute("job.outcome", state)
if state == "ready":
outcome = "success"
jobs_processed.add(
1,
{
"job_type": "image",
"outcome": "success",
},
)
elif state == "failed":
outcome = "failed"
span.set_status(
Status(
StatusCode.ERROR,
"Background job processing failed",
)
)
jobs_failed.add(
1,
{
"job_type": "image",
"outcome": "failed",
},
)
elif state in {"retry", "busy", "deferred"}:
outcome = "retry"
else:
outcome = "other"
return result
except Exception as exc:
outcome = "failed"
span.set_attribute("job.outcome", "failed")
span.record_exception(exc)
span.set_status(
Status(
StatusCode.ERROR,
"Background job processing failed",
)
)
jobs_failed.add(
1,
{
"job_type": "image",
"outcome": "failed",
},
)
raise
finally:
elapsed = time.perf_counter() - started
job_duration.record(
elapsed,
{
"job_type": "image",
"outcome": outcome,
},
)