forked from mtusi2026/mtusi2026lab6
240 lines
6.4 KiB
Python
240 lines
6.4 KiB
Python
import logging
|
|
import os
|
|
import socket
|
|
import time
|
|
|
|
from opentelemetry import metrics, trace
|
|
from opentelemetry._logs import set_logger_provider
|
|
from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
|
|
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
|
OTLPMetricExporter,
|
|
)
|
|
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
|
OTLPSpanExporter,
|
|
)
|
|
from opentelemetry.instrumentation.logging import LoggingInstrumentor
|
|
from opentelemetry.propagate import extract
|
|
from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
|
|
from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
|
|
from opentelemetry.sdk.metrics import MeterProvider
|
|
from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
|
|
from opentelemetry.sdk.resources import Resource
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import BatchSpanProcessor
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_initialized = False
|
|
|
|
tracer = None
|
|
jobs_processed = None
|
|
jobs_failed = None
|
|
job_duration = None
|
|
|
|
|
|
def setup_worker_telemetry():
|
|
"""Initialize OpenTelemetry for the background worker."""
|
|
global _initialized
|
|
global tracer, jobs_processed, jobs_failed, job_duration
|
|
|
|
if _initialized:
|
|
return
|
|
|
|
endpoint = os.getenv(
|
|
"OTEL_EXPORTER_OTLP_ENDPOINT",
|
|
"http://otel-collector:4318",
|
|
).rstrip("/")
|
|
|
|
service_name = os.getenv(
|
|
"OTEL_SERVICE_NAME",
|
|
"messenger-worker",
|
|
)
|
|
|
|
resource = Resource.create(
|
|
{
|
|
"service.name": service_name,
|
|
"service.version": "1.4.0",
|
|
"service.instance.id": (
|
|
os.getenv("INSTANCE_ID")
|
|
or socket.gethostname()
|
|
),
|
|
"deployment.environment.name": os.getenv(
|
|
"APP_ENV",
|
|
"lab",
|
|
),
|
|
}
|
|
)
|
|
|
|
# Distributed tracing.
|
|
tracer_provider = TracerProvider(resource=resource)
|
|
tracer_provider.add_span_processor(
|
|
BatchSpanProcessor(
|
|
OTLPSpanExporter(
|
|
endpoint=f"{endpoint}/v1/traces",
|
|
)
|
|
)
|
|
)
|
|
trace.set_tracer_provider(tracer_provider)
|
|
|
|
# Metrics.
|
|
metric_reader = PeriodicExportingMetricReader(
|
|
OTLPMetricExporter(
|
|
endpoint=f"{endpoint}/v1/metrics",
|
|
),
|
|
export_interval_millis=10000,
|
|
)
|
|
|
|
meter_provider = MeterProvider(
|
|
resource=resource,
|
|
metric_readers=[metric_reader],
|
|
)
|
|
metrics.set_meter_provider(meter_provider)
|
|
|
|
tracer = trace.get_tracer("messenger.worker")
|
|
meter = metrics.get_meter("messenger.worker")
|
|
|
|
jobs_processed = meter.create_counter(
|
|
name="jobs_processed",
|
|
description="Number of successfully processed background jobs",
|
|
unit="1",
|
|
)
|
|
|
|
jobs_failed = meter.create_counter(
|
|
name="jobs_failed",
|
|
description="Number of failed background jobs",
|
|
unit="1",
|
|
)
|
|
|
|
job_duration = meter.create_histogram(
|
|
name="job_processing_duration",
|
|
description="Background job processing duration",
|
|
unit="s",
|
|
)
|
|
|
|
# Logs.
|
|
logger_provider = LoggerProvider(resource=resource)
|
|
logger_provider.add_log_record_processor(
|
|
BatchLogRecordProcessor(
|
|
OTLPLogExporter(
|
|
endpoint=f"{endpoint}/v1/logs",
|
|
)
|
|
)
|
|
)
|
|
set_logger_provider(logger_provider)
|
|
|
|
# Add trace_id and span_id to Python log records.
|
|
LoggingInstrumentor().instrument(
|
|
tracer_provider=tracer_provider,
|
|
set_logging_format=False,
|
|
)
|
|
|
|
# Export Python logging records through OTLP.
|
|
logging_handler = LoggingHandler(
|
|
level=logging.INFO,
|
|
logger_provider=logger_provider,
|
|
)
|
|
logging.getLogger().addHandler(logging_handler)
|
|
|
|
_initialized = True
|
|
|
|
logger.info(
|
|
"OpenTelemetry initialized for worker service %s",
|
|
service_name,
|
|
)
|
|
|
|
|
|
def process_job_with_telemetry(process_function, payload):
|
|
"""Run one job inside a consumer span and record its metrics."""
|
|
if not _initialized:
|
|
setup_worker_telemetry()
|
|
|
|
started = time.perf_counter()
|
|
outcome = "other"
|
|
|
|
# Continue the trace started by the API request.
|
|
context = extract(payload.get("trace_context", {}))
|
|
|
|
with tracer.start_as_current_span(
|
|
"worker.process_job",
|
|
context=context,
|
|
kind=SpanKind.CONSUMER,
|
|
attributes={
|
|
"messaging.system": "rabbitmq",
|
|
"messaging.operation": "process",
|
|
"job.type": "image",
|
|
},
|
|
) as span:
|
|
try:
|
|
result = process_function(payload)
|
|
|
|
state = result.get("state", "unknown")
|
|
span.set_attribute("job.outcome", state)
|
|
|
|
if state == "ready":
|
|
outcome = "success"
|
|
jobs_processed.add(
|
|
1,
|
|
{
|
|
"job_type": "image",
|
|
"outcome": "success",
|
|
},
|
|
)
|
|
|
|
elif state == "failed":
|
|
outcome = "failed"
|
|
span.set_status(
|
|
Status(
|
|
StatusCode.ERROR,
|
|
"Background job processing failed",
|
|
)
|
|
)
|
|
jobs_failed.add(
|
|
1,
|
|
{
|
|
"job_type": "image",
|
|
"outcome": "failed",
|
|
},
|
|
)
|
|
|
|
elif state in {"retry", "busy", "deferred"}:
|
|
outcome = "retry"
|
|
|
|
else:
|
|
outcome = "other"
|
|
|
|
return result
|
|
|
|
except Exception as exc:
|
|
outcome = "failed"
|
|
span.set_attribute("job.outcome", "failed")
|
|
span.record_exception(exc)
|
|
span.set_status(
|
|
Status(
|
|
StatusCode.ERROR,
|
|
"Background job processing failed",
|
|
)
|
|
)
|
|
|
|
jobs_failed.add(
|
|
1,
|
|
{
|
|
"job_type": "image",
|
|
"outcome": "failed",
|
|
},
|
|
)
|
|
|
|
raise
|
|
|
|
finally:
|
|
elapsed = time.perf_counter() - started
|
|
|
|
job_duration.record(
|
|
elapsed,
|
|
{
|
|
"job_type": "image",
|
|
"outcome": outcome,
|
|
},
|
|
)
|