Files
mtusi2026lab6/internal/telemetry/telemetry.go
T

243 lines
11 KiB
Go

// Package telemetry owns bounded, best-effort OTLP exporters. It never affects
// business readiness and never exports URLs, SQL text, credentials or payloads.
package telemetry
import (
"context"
"fmt"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/contrib/bridges/otelslog"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.32.0"
"go.opentelemetry.io/otel/trace"
"log/slog"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
)
var HTTPCount metric.Int64Counter
var HTTPDuration, JobDuration, ReadyDuration metric.Float64Histogram
var Jobs metric.Int64Counter
var PendingAcquires atomic.Int64
var Busy atomic.Int64
var capacity atomic.Int64
var tracer = otel.Tracer("messenger", trace.WithSchemaURL(semconv.SchemaURL))
func Init(ctx context.Context, service string) (*slog.Logger, func(), error) {
instance, _ := os.Hostname()
attrs := []attribute.KeyValue{attribute.String("service.name", service), attribute.String("service.version", "6.1.0"), attribute.String("service.instance.id", instance), attribute.String("deployment.environment.name", "lab")}
res := resource.NewWithAttributes(semconv.SchemaURL, attrs...)
otel.SetTextMapPropagator(propagation.TraceContext{})
if os.Getenv("OTEL_SDK_DISABLED") == "true" {
return slog.New(&contextHandler{next: slog.NewJSONHandler(os.Stdout, nil), service: service}), func() {}, nil
}
te, e := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(time.Second))
if e != nil {
return nil, nil, e
}
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(1))), sdktrace.WithBatcher(te, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second)))
otel.SetTracerProvider(tp)
tracer = otel.Tracer("messenger", trace.WithSchemaURL(semconv.SchemaURL))
me, e := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(time.Second))
if e != nil {
return nil, nil, e
}
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me, sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(time.Second))), sdkmetric.WithView(sdkmetric.NewView(sdkmetric.Instrument{Kind: sdkmetric.InstrumentKindHistogram}, sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{Boundaries: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 30, 60, 120}}})))
otel.SetMeterProvider(mp)
le, e := otlploghttp.New(ctx, otlploghttp.WithTimeout(time.Second))
if e != nil {
return nil, nil, e
}
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(le, sdklog.WithMaxQueueSize(1024), sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second))))
log := slog.New(&contextHandler{next: &tee{a: slog.NewJSONHandler(os.Stdout, nil), b: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp))}, service: service})
m := otel.Meter("messenger")
HTTPCount, _ = m.Int64Counter("http.server.requests")
HTTPDuration, _ = m.Float64Histogram("http.server.duration", metric.WithUnit("s"))
Jobs, _ = m.Int64Counter("messenger.jobs")
JobDuration, _ = m.Float64Histogram("messenger.job.duration", metric.WithUnit("s"))
ReadyDuration, _ = m.Float64Histogram("messenger.attachment.ready.duration", metric.WithUnit("s"))
shutdown := func() {
c, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
_ = tp.Shutdown(c)
_ = mp.Shutdown(c)
_ = lp.Shutdown(c)
}
return log, shutdown, nil
}
func Start(ctx context.Context, name string, kind trace.SpanKind) (context.Context, trace.Span) {
return tracer.Start(ctx, name, trace.WithSpanKind(kind))
}
func Failure(span trace.Span, e error) {
if e != nil {
span.SetStatus(codes.Error, "dependency operation failed")
}
}
func Inject(ctx context.Context) (string, string) {
c := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, c)
return c.Get("traceparent"), c.Get("tracestate")
}
func Extract(ctx context.Context, parent, state string) context.Context {
return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state})
}
func TraceID(ctx context.Context) string { return trace.SpanContextFromContext(ctx).TraceID().String() }
func HTTP(ctx context.Context, method, route string, status int, seconds float64) {
switch method {
case "GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS":
default:
method = "_OTHER"
}
if HTTPCount == nil {
return
}
attrs := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", fmt.Sprintf("%dxx", status/100)))
HTTPCount.Add(ctx, 1, attrs)
HTTPDuration.Record(ctx, seconds, attrs)
}
func Job(ctx context.Context, outcome string, seconds float64) {
if Jobs == nil {
return
}
attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome))
Jobs.Add(ctx, 1, attrs)
JobDuration.Record(ctx, seconds, attrs)
}
func Ready(ctx context.Context, seconds float64) {
if ReadyDuration != nil {
ReadyDuration.Record(ctx, seconds)
}
}
func WorkerCapacity(n int) { capacity.Store(int64(n)) }
func Observe(pool *pgxpool.Pool, worker bool) {
m := otel.Meter("messenger")
used, _ := m.Int64ObservableGauge("messenger.db.pool.used")
limit, _ := m.Int64ObservableGauge("messenger.db.pool.limit")
waiting, _ := m.Int64ObservableGauge("messenger.db.pool.waiting")
waits, _ := m.Int64ObservableCounter("messenger.db.pool.waits")
depth, _ := m.Int64ObservableGauge("messenger.queue.depth")
age, _ := m.Float64ObservableGauge("messenger.queue.oldest.age", metric.WithUnit("s"))
busy, _ := m.Int64ObservableGauge("messenger.worker.busy")
cap, _ := m.Int64ObservableGauge("messenger.worker.capacity")
sliTotal, _ := m.Int64ObservableCounter("messenger.attachment.sli.total")
sliBad, _ := m.Int64ObservableCounter("messenger.attachment.sli.bad")
containerRSS, _ := m.Int64ObservableGauge("container.memory.usage", metric.WithUnit("By"))
containerCPU, _ := m.Float64ObservableCounter("container.cpu.time", metric.WithUnit("s"))
rss, _ := m.Int64ObservableGauge("process.memory.usage", metric.WithUnit("By"))
cpu, _ := m.Float64ObservableCounter("process.cpu.time", metric.WithUnit("s"))
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
stat := pool.Stat()
o.ObserveInt64(used, int64(stat.AcquiredConns()))
o.ObserveInt64(limit, int64(stat.MaxConns()))
o.ObserveInt64(waits, stat.EmptyAcquireCount())
o.ObserveInt64(waiting, PendingAcquires.Load())
o.ObserveInt64(busy, Busy.Load())
o.ObserveInt64(cap, capacity.Load())
mem, cputime := processUsage()
o.ObserveInt64(rss, mem)
o.ObserveFloat64(cpu, cputime)
if raw, e := os.ReadFile("/sys/fs/cgroup/memory.current"); e == nil {
if bytes, e := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64); e == nil {
o.ObserveInt64(containerRSS, bytes)
}
}
if raw, e := os.ReadFile("/sys/fs/cgroup/cpu.stat"); e == nil {
for _, line := range strings.Split(string(raw), "\n") {
fields := strings.Fields(line)
if len(fields) == 2 && fields[0] == "usage_usec" {
if us, e := strconv.ParseFloat(fields[1], 64); e == nil {
o.ObserveFloat64(containerCPU, us/1e6)
}
}
}
}
if worker {
// One bounded query on the worker only avoids duplicating global queue
// gauges per API. A DB failure omits the observation, never invents zero.
query, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
defer cancel()
var n int64
var oldest float64
e := pool.QueryRow(query, `SELECT count(*),COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(created_at)),0)::float8 FROM pipeline_jobs WHERE finished_at IS NULL`).Scan(&n, &oldest)
if e == nil {
o.ObserveInt64(depth, n)
o.ObserveFloat64(age, oldest)
}
var matured, bad int64
e = pool.QueryRow(query, `SELECT count(*),count(*) FILTER (WHERE ready_at IS NULL OR ready_at>created_at+interval '5 seconds') FROM pipeline_jobs WHERE created_at<=clock_timestamp()-interval '5 seconds'`).Scan(&matured, &bad)
if e == nil {
o.ObserveInt64(sliTotal, matured)
o.ObserveInt64(sliBad, bad)
}
}
return nil
}, used, limit, waits, waiting, depth, age, busy, cap, rss, cpu, sliTotal, sliBad, containerRSS, containerCPU)
}
func processUsage() (int64, float64) {
raw, e := os.ReadFile("/proc/self/stat")
if e != nil {
return 0, 0
}
end := strings.LastIndex(string(raw), ")")
if end < 0 {
return 0, 0
}
fields := strings.Fields(string(raw)[end+1:])
if len(fields) < 22 {
return 0, 0
}
u, _ := strconv.ParseFloat(fields[11], 64)
s, _ := strconv.ParseFloat(fields[12], 64)
pages, _ := strconv.ParseInt(fields[21], 10, 64)
return pages * int64(os.Getpagesize()), (u + s) / 100
}
type contextHandler struct {
next slog.Handler
service string
}
func (h *contextHandler) Enabled(ctx context.Context, l slog.Level) bool {
return h.next.Enabled(ctx, l)
}
func (h *contextHandler) Handle(ctx context.Context, r slog.Record) error {
r.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message))
sc := trace.SpanContextFromContext(ctx)
if sc.IsValid() {
r.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String()))
}
return h.next.Handle(ctx, r)
}
func (h *contextHandler) WithAttrs(a []slog.Attr) slog.Handler {
return &contextHandler{next: h.next.WithAttrs(a), service: h.service}
}
func (h *contextHandler) WithGroup(g string) slog.Handler {
return &contextHandler{next: h.next.WithGroup(g), service: h.service}
}
type tee struct{ a, b slog.Handler }
func (h *tee) Enabled(ctx context.Context, l slog.Level) bool {
return h.a.Enabled(ctx, l) || h.b.Enabled(ctx, l)
}
func (h *tee) Handle(ctx context.Context, r slog.Record) error {
_ = h.a.Handle(ctx, r.Clone())
return h.b.Handle(ctx, r)
}
func (h *tee) WithAttrs(a []slog.Attr) slog.Handler { return &tee{h.a.WithAttrs(a), h.b.WithAttrs(a)} }
func (h *tee) WithGroup(g string) slog.Handler { return &tee{h.a.WithGroup(g), h.b.WithGroup(g)} }