Files
mtusi2026lab6-fork/telemetry.go

353 lines
12 KiB
Go

package main
import (
"context"
"log/slog"
"os"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/contrib/bridges/otelslog"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
const appVersion = "1.6.0"
var (
tracer = otel.Tracer("messenger")
meter = otel.Meter("messenger")
httpRequests = mustCounter("http.server.requests", "{request}", "HTTP requests by method, route template and status class")
httpDuration = mustHistogram("http.server.request.duration", "s", "HTTP server request duration",
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
jobsProcessed = mustCounter("jobs.processed", "{job}", "Processed job attempts by outcome")
jobDuration = mustHistogram("job.processing.duration", "s", "Job processing duration",
0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
jobTimeToReady = mustHistogram("job.time_to_ready", "s", "Time from durable acceptance to ready",
0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300)
exportErrors atomic.Int64
)
type exportErrorHandler struct{}
func (exportErrorHandler) Handle(error) { exportErrors.Add(1) }
func mustCounter(name, unit, desc string) metric.Int64Counter {
c, err := meter.Int64Counter(name, metric.WithUnit(unit), metric.WithDescription(desc))
if err != nil {
panic(err)
}
return c
}
func mustHistogram(name, unit, desc string, buckets ...float64) metric.Float64Histogram {
h, err := meter.Float64Histogram(name, metric.WithUnit(unit), metric.WithDescription(desc),
metric.WithExplicitBucketBoundaries(buckets...))
if err != nil {
panic(err)
}
return h
}
type fanoutHandler struct {
stdout, remote slog.Handler
service string
}
func (h fanoutHandler) Enabled(ctx context.Context, l slog.Level) bool {
return h.stdout.Enabled(ctx, l)
}
func (h fanoutHandler) Handle(ctx context.Context, r slog.Record) error {
local := r.Clone()
local.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message))
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
local.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String()))
}
err := h.stdout.Handle(ctx, local)
remote := r.Clone()
remote.AddAttrs(slog.String("event", r.Message))
_ = h.remote.Handle(ctx, remote)
return err
}
func (h fanoutHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
return fanoutHandler{stdout: h.stdout.WithAttrs(attrs), remote: h.remote.WithAttrs(attrs), service: h.service}
}
func (h fanoutHandler) WithGroup(name string) slog.Handler {
return fanoutHandler{stdout: h.stdout.WithGroup(name), remote: h.remote.WithGroup(name), service: h.service}
}
func setupTelemetry(ctx context.Context, defaultService string) (*slog.Logger, func(context.Context)) {
service := envOr("OTEL_SERVICE_NAME", defaultService)
res, err := resource.New(ctx,
resource.WithAttributes(
attribute.String("service.name", service),
attribute.String("service.version", appVersion),
attribute.String("service.instance.id", instanceID),
attribute.String("deployment.environment.name", envOr("APP_ENV", "lab")),
),
resource.WithFromEnv(),
)
if err != nil {
panic(err)
}
otel.SetErrorHandler(exportErrorHandler{})
otel.SetTextMapPropagator(propagation.TraceContext{})
traceExp, err := otlptracehttp.New(ctx,
otlptracehttp.WithTimeout(3*time.Second),
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
if err != nil {
panic(err)
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithResource(res),
sdktrace.WithBatcher(traceExp,
sdktrace.WithBatchTimeout(time.Second),
sdktrace.WithMaxQueueSize(2048),
sdktrace.WithMaxExportBatchSize(512),
sdktrace.WithExportTimeout(5*time.Second)),
)
otel.SetTracerProvider(tp)
metricExp, err := otlpmetrichttp.New(ctx,
otlpmetrichttp.WithTimeout(3*time.Second),
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
if err != nil {
panic(err)
}
mp := sdkmetric.NewMeterProvider(
sdkmetric.WithResource(res),
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExp, sdkmetric.WithInterval(10*time.Second), sdkmetric.WithTimeout(5*time.Second))),
)
otel.SetMeterProvider(mp)
logExp, err := otlploghttp.New(ctx,
otlploghttp.WithTimeout(3*time.Second),
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
if err != nil {
panic(err)
}
lp := sdklog.NewLoggerProvider(
sdklog.WithResource(res),
sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp,
sdklog.WithMaxQueueSize(2048),
sdklog.WithExportInterval(time.Second),
sdklog.WithExportTimeout(5*time.Second))),
)
registerProcessMetrics()
log := slog.New(fanoutHandler{
stdout: slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}),
remote: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp)),
service: service,
})
shutdown := func(ctx context.Context) {
_ = tp.Shutdown(ctx)
_ = mp.Shutdown(ctx)
_ = lp.Shutdown(ctx)
}
return log, shutdown
}
func observableGauge(name, unit, desc string, cb func(context.Context, metric.Int64Observer) error) {
_, err := meter.Int64ObservableGauge(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithInt64Callback(cb))
if err != nil {
panic(err)
}
}
func observableCounter(name, unit, desc string, cb func(context.Context, metric.Float64Observer) error) {
_, err := meter.Float64ObservableCounter(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithFloat64Callback(cb))
if err != nil {
panic(err)
}
}
func registerProcessMetrics() {
observableCounter("process.cpu.time", "s", "Process CPU time", func(_ context.Context, o metric.Float64Observer) error {
var ru syscall.Rusage
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil {
return nil
}
sec := func(t syscall.Timeval) float64 { return float64(t.Sec) + float64(t.Usec)/1e6 }
o.Observe(sec(ru.Utime) + sec(ru.Stime))
return nil
})
observableGauge("process.memory.usage", "By", "Resident set size", func(_ context.Context, o metric.Int64Observer) error {
data, err := os.ReadFile("/proc/self/status")
if err != nil {
return nil
}
for _, line := range strings.Split(string(data), "\n") {
if strings.HasPrefix(line, "VmRSS:") {
f := strings.Fields(line)
if len(f) >= 2 {
var kb int64
for _, c := range f[1] {
kb = kb*10 + int64(c-'0')
}
o.Observe(kb * 1024)
}
}
}
return nil
})
observableCounter("telemetry.export.errors", "{error}", "Failed telemetry exports", func(_ context.Context, o metric.Float64Observer) error {
o.Observe(float64(exportErrors.Load()))
return nil
})
}
func registerPoolMetrics(db *pgxpool.Pool) {
observableGauge("db.pool.connections", "{connection}", "DB pool connections by state", func(_ context.Context, o metric.Int64Observer) error {
s := db.Stat()
o.Observe(int64(s.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used")))
o.Observe(int64(s.IdleConns()), metric.WithAttributes(attribute.String("state", "idle")))
return nil
})
observableGauge("db.pool.max_connections", "{connection}", "DB pool size limit", func(_ context.Context, o metric.Int64Observer) error {
o.Observe(int64(db.Stat().MaxConns()))
return nil
})
observableCounter("db.pool.empty_acquires", "{acquire}", "Acquires that had to wait for a connection", func(_ context.Context, o metric.Float64Observer) error {
o.Observe(float64(db.Stat().EmptyAcquireCount()))
return nil
})
observableCounter("db.pool.acquire.wait", "s", "Total time spent acquiring connections", func(_ context.Context, o metric.Float64Observer) error {
o.Observe(db.Stat().AcquireDuration().Seconds())
return nil
})
}
func registerQueueMetrics(db *pgxpool.Pool) {
var mu sync.Mutex
var depth int64
var age float64
refresh := func(ctx context.Context) {
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
var d int64
var a float64
err := db.QueryRow(ctx, `
select count(*),
coalesce(extract(epoch from now() - min(next_attempt_at) filter (where next_attempt_at <= now())), 0)
from jobs where status = 'queued'`).Scan(&d, &a)
if err != nil {
return
}
mu.Lock()
depth, age = d, a
mu.Unlock()
}
attrs := metric.WithAttributes(attribute.String("queue", jobStream))
observableGauge("queue.depth", "{job}", "Queued jobs", func(ctx context.Context, o metric.Int64Observer) error {
refresh(ctx)
mu.Lock()
defer mu.Unlock()
o.Observe(depth, attrs)
return nil
})
_, err := meter.Float64ObservableGauge("queue.oldest_job_age", metric.WithUnit("s"), metric.WithDescription("Age of the oldest available queued job"),
metric.WithFloat64Callback(func(_ context.Context, o metric.Float64Observer) error {
mu.Lock()
defer mu.Unlock()
o.Observe(age, attrs)
return nil
}))
if err != nil {
panic(err)
}
}
func registerWorkerMetrics(w *Worker, concurrency int) {
observableGauge("worker.busy", "{slot}", "Jobs being processed", func(_ context.Context, o metric.Int64Observer) error {
o.Observe(w.busy.Load())
return nil
})
observableGauge("worker.capacity", "{slot}", "Configured worker concurrency", func(_ context.Context, o metric.Int64Observer) error {
o.Observe(int64(concurrency))
return nil
})
}
func childSpan(ctx context.Context, name string, kind trace.SpanKind, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
if !trace.SpanContextFromContext(ctx).IsValid() {
return ctx, trace.SpanFromContext(ctx)
}
return tracer.Start(ctx, name, trace.WithSpanKind(kind), trace.WithAttributes(attrs...))
}
func endSpan(sp trace.Span, err error) {
if err != nil {
sp.RecordError(err)
sp.SetStatus(codes.Error, err.Error())
}
sp.End()
}
type traceCarrier struct{ traceparent, tracestate string }
func injectTrace(ctx context.Context) traceCarrier {
m := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, m)
return traceCarrier{traceparent: m.Get("traceparent"), tracestate: m.Get("tracestate")}
}
func extractTrace(ctx context.Context, c traceCarrier) context.Context {
if c.traceparent == "" {
return ctx
}
m := propagation.MapCarrier{"traceparent": c.traceparent}
if c.tracestate != "" {
m["tracestate"] = c.tracestate
}
return otel.GetTextMapPropagator().Extract(ctx, m)
}
type dbTracer struct{}
type dbSpanKey struct{}
func (dbTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
if !trace.SpanContextFromContext(ctx).IsValid() {
return ctx
}
op := "query"
if f := strings.Fields(data.SQL); len(f) > 0 {
op = strings.ToUpper(f[0])
}
ctx, sp := tracer.Start(ctx, "db "+op, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(
attribute.String("db.system.name", "postgresql"),
attribute.String("db.operation.name", op),
attribute.String("db.query.text", data.SQL),
))
return context.WithValue(ctx, dbSpanKey{}, sp)
}
func (dbTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
if sp, ok := ctx.Value(dbSpanKey{}).(trace.Span); ok {
endSpan(sp, data.Err)
}
}