Files
mtusi2026lab6/telemetry.go
T
2026-11-19 19:24:45 +03:00

274 lines
10 KiB
Go

package main
import (
"context"
"errors"
"fmt"
"log/slog"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
const telemetryScope = "mtusi/messenger"
var traceContext propagation.TraceContext
type telemetry struct {
logger otellog.Logger
requests metric.Int64Counter
httpTime metric.Float64Histogram
jobs metric.Int64Counter
jobTime metric.Float64Histogram
readyTime metric.Float64Histogram
workerBusy atomic.Int64
}
func startTelemetry(role string, db *pgxpool.Pool, poolTrace *databaseTracer) (*telemetry, func(), error) {
if env("OTEL_SDK_DISABLED", "false") == "true" {
return nil, func() {}, nil
}
ctx := context.Background()
host, err := os.Hostname()
if err != nil {
return nil, nil, errors.New("cannot identify telemetry instance")
}
res := resource.NewSchemaless(
attribute.String("service.name", "messenger-"+role),
attribute.String("service.version", "6.0.0"),
attribute.String("service.instance.id", host),
attribute.String("deployment.environment.name", env("APP_ENV", "lab")),
)
const exportTimeout = 2 * time.Second
tracer, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(exportTimeout),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
if err != nil {
return nil, nil, errors.New("invalid OTLP trace configuration")
}
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.AlwaysSample()),
sdktrace.WithBatcher(tracer, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128),
sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(exportTimeout)))
var mp *sdkmetric.MeterProvider
var lp *sdklog.LoggerProvider
shutdown := func() {
end, cancel := context.WithTimeout(context.Background(), exportTimeout)
defer cancel()
if lp != nil {
_ = lp.Shutdown(end)
}
if mp != nil {
_ = mp.Shutdown(end)
}
_ = tp.Shutdown(end)
}
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(exportTimeout),
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
if err != nil {
shutdown()
return nil, nil, errors.New("invalid OTLP metric configuration")
}
reader := sdkmetric.NewPeriodicReader(metrics, sdkmetric.WithInterval(2*time.Second),
sdkmetric.WithTimeout(exportTimeout))
mp = sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(reader))
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(exportTimeout),
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
if err != nil {
shutdown()
return nil, nil, errors.New("invalid OTLP log configuration")
}
lp = sdklog.NewLoggerProvider(sdklog.WithResource(res),
sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(1024),
sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second),
sdklog.WithExportBufferSize(1), sdklog.WithExportTimeout(exportTimeout))))
t, err := makeTelemetry(mp.Meter(telemetryScope))
if err == nil {
err = observeRuntime(mp.Meter(telemetryScope), role, db, poolTrace)
}
if err == nil && role == "worker" {
err = t.observeWorker(mp.Meter(telemetryScope))
}
if err != nil {
shutdown()
return nil, nil, fmt.Errorf("register telemetry instruments: %w", err)
}
t.logger = lp.Logger(telemetryScope)
otel.SetTracerProvider(tp)
// Exporter errors can contain the configured endpoint. Local diagnostics
// report the fixed event only, and never feed it back into the OTLP queue.
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { slog.Warn("telemetry export failed") }))
return t, shutdown, nil
}
func makeTelemetry(m metric.Meter) (*telemetry, error) {
t := &telemetry{}
var err, errHTTP, errJobs, errJobTime, errReady error
t.requests, err = m.Int64Counter("messenger_http_requests")
t.httpTime, errHTTP = m.Float64Histogram("messenger_http_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.002, .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30))
t.jobs, errJobs = m.Int64Counter("messenger_job_attempts")
t.jobTime, errJobTime = m.Float64Histogram("messenger_job_duration", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120))
t.readyTime, errReady = m.Float64Histogram("messenger_attachment_time_to_ready", metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(.25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300))
return t, errors.Join(err, errHTTP, errJobs, errJobTime, errReady)
}
func (t *telemetry) event(ctx context.Context, event string, level otellog.Severity, fields ...otellog.KeyValue) {
if t == nil {
return
}
var record otellog.Record
record.SetTimestamp(time.Now())
record.SetSeverity(level)
record.SetSeverityText(level.String())
record.SetBody(otellog.StringValue(event))
record.AddAttributes(otellog.String("event", event))
record.AddAttributes(fields...)
t.logger.Emit(ctx, record)
}
func (t *telemetry) finishHTTP(ctx context.Context, route, method, requestID string, status int, start time.Time) {
if t == nil {
return
}
class := strconv.Itoa(status/100) + "xx"
dimensions := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
attribute.String("status_class", class))
duration := time.Since(start).Seconds()
t.requests.Add(ctx, 1, dimensions)
if class != "5xx" {
t.requests.Add(ctx, 0, metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
attribute.String("status_class", "5xx")))
}
t.httpTime.Record(ctx, duration, dimensions)
span := trace.SpanFromContext(ctx)
span.SetAttributes(attribute.Int("http.response.status_code", status))
severity := otellog.SeverityInfo
if status >= 500 {
span.SetStatus(codes.Error, "server_error")
severity = otellog.SeverityError
}
t.event(ctx, "http.request", severity, otellog.String("request_id", requestID),
otellog.String("route", route), otellog.String("method", method), otellog.Int("status", status),
otellog.Float64("duration_seconds", duration))
}
func (t *telemetry) finishJob(ctx context.Context, j imageJob, outcome string, started time.Time) {
if t == nil {
return
}
duration := time.Since(started).Seconds()
dimensions := metric.WithAttributes(attribute.String("job_type", j.Type), attribute.String("outcome", outcome))
t.jobs.Add(ctx, 1, dimensions)
t.jobTime.Record(ctx, duration, dimensions)
severity := otellog.SeverityInfo
if outcome != "ready" {
trace.SpanFromContext(ctx).SetStatus(codes.Error, outcome)
severity = otellog.SeverityError
}
t.event(ctx, "image.attempt", severity, otellog.String("job_id", j.JobID),
otellog.Int("attempt", j.Attempt), otellog.String("outcome", outcome),
otellog.Float64("duration_seconds", duration))
}
func safeHTTPRoute(method, path string) (string, string) {
switch method {
case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS":
default:
method = "OTHER"
}
switch path {
case "/api/v1/users", "/api/v1/users/me", "/api/v1/auth/sessions",
"/api/v1/auth/sessions/current", "/api/v1/chats":
return method, path
}
parts := strings.SplitN(path, "/", 7)
if len(parts) >= 5 && parts[0] == "" && parts[1] == "api" && parts[2] == "v1" {
switch parts[3] {
case "chats":
if len(parts) == 5 {
return method, "/api/v1/chats/{chat_id}"
}
if len(parts) == 6 && (parts[5] == "messages" || parts[5] == "attachments") {
return method, "/api/v1/chats/{chat_id}/" + parts[5]
}
case "attachments":
if len(parts) == 5 {
return method, "/api/v1/attachments/{attachment_id}"
}
if len(parts) == 6 && parts[5] == "content" {
return method, "/api/v1/attachments/{attachment_id}/content"
}
}
}
return method, "unmatched"
}
func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
return otel.Tracer(telemetryScope).Start(ctx, name, opts...)
}
func endSpan(span trace.Span, err error) {
if err != nil {
span.SetStatus(codes.Error, "operation_failed")
}
span.End()
}
func jobSpanAttributes(j imageJob) trace.SpanStartOption {
return trace.WithAttributes(attribute.String("messaging.system", "valkey"),
attribute.String("messaging.destination.name", imageStream),
attribute.String("messaging.message.id", j.JobID), attribute.Int("job.attempt", j.Attempt))
}
func (t *telemetry) startJob(ctx context.Context, original propagation.MapCarrier, j imageJob, producer bool) (context.Context, trace.Span) {
if t != nil {
ctx = traceContext.Extract(ctx, original)
}
if producer {
return startSpan(ctx, "attachment-jobs publish", trace.WithSpanKind(trace.SpanKindProducer), jobSpanAttributes(j))
}
return startSpan(ctx, "attachment-jobs process", trace.WithSpanKind(trace.SpanKindConsumer), jobSpanAttributes(j))
}
// pgx receives full SQL and bind values. Export a fixed span name and status
// only, and skip calls outside a traced request or consumer.
func (*databaseTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
if !trace.SpanFromContext(ctx).IsRecording() {
return ctx
}
child, span := startSpan(ctx, "postgresql query", trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "postgresql")))
return context.WithValue(child, querySpanKey{}, span)
}
type querySpanKey struct{}
func (*databaseTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
if span, ok := ctx.Value(querySpanKey{}).(trace.Span); ok {
if errors.Is(data.Err, pgx.ErrNoRows) {
data.Err = nil
}
endSpan(span, data.Err)
}
}