243 lines
11 KiB
Go
243 lines
11 KiB
Go
// Package telemetry owns bounded, best-effort OTLP exporters. It never affects
|
|
// business readiness and never exports URLs, SQL text, credentials or payloads.
|
|
package telemetry
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.opentelemetry.io/contrib/bridges/otelslog"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdklog "go.opentelemetry.io/otel/sdk/log"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
semconv "go.opentelemetry.io/otel/semconv/v1.32.0"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"log/slog"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
)
|
|
|
|
var HTTPCount metric.Int64Counter
|
|
var HTTPDuration, JobDuration, ReadyDuration metric.Float64Histogram
|
|
var Jobs metric.Int64Counter
|
|
var PendingAcquires atomic.Int64
|
|
var Busy atomic.Int64
|
|
var capacity atomic.Int64
|
|
var tracer = otel.Tracer("messenger", trace.WithSchemaURL(semconv.SchemaURL))
|
|
|
|
func Init(ctx context.Context, service string) (*slog.Logger, func(), error) {
|
|
instance, _ := os.Hostname()
|
|
attrs := []attribute.KeyValue{attribute.String("service.name", service), attribute.String("service.version", "6.1.0"), attribute.String("service.instance.id", instance), attribute.String("deployment.environment.name", "lab")}
|
|
res := resource.NewWithAttributes(semconv.SchemaURL, attrs...)
|
|
otel.SetTextMapPropagator(propagation.TraceContext{})
|
|
if os.Getenv("OTEL_SDK_DISABLED") == "true" {
|
|
return slog.New(&contextHandler{next: slog.NewJSONHandler(os.Stdout, nil), service: service}), func() {}, nil
|
|
}
|
|
te, e := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(time.Second))
|
|
if e != nil {
|
|
return nil, nil, e
|
|
}
|
|
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(1))), sdktrace.WithBatcher(te, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second)))
|
|
otel.SetTracerProvider(tp)
|
|
tracer = otel.Tracer("messenger", trace.WithSchemaURL(semconv.SchemaURL))
|
|
me, e := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(time.Second))
|
|
if e != nil {
|
|
return nil, nil, e
|
|
}
|
|
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(me, sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(time.Second))), sdkmetric.WithView(sdkmetric.NewView(sdkmetric.Instrument{Kind: sdkmetric.InstrumentKindHistogram}, sdkmetric.Stream{Aggregation: sdkmetric.AggregationExplicitBucketHistogram{Boundaries: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 20, 30, 60, 120}}})))
|
|
otel.SetMeterProvider(mp)
|
|
le, e := otlploghttp.New(ctx, otlploghttp.WithTimeout(time.Second))
|
|
if e != nil {
|
|
return nil, nil, e
|
|
}
|
|
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(le, sdklog.WithMaxQueueSize(1024), sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second))))
|
|
log := slog.New(&contextHandler{next: &tee{a: slog.NewJSONHandler(os.Stdout, nil), b: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp))}, service: service})
|
|
m := otel.Meter("messenger")
|
|
HTTPCount, _ = m.Int64Counter("http.server.requests")
|
|
HTTPDuration, _ = m.Float64Histogram("http.server.duration", metric.WithUnit("s"))
|
|
Jobs, _ = m.Int64Counter("messenger.jobs")
|
|
JobDuration, _ = m.Float64Histogram("messenger.job.duration", metric.WithUnit("s"))
|
|
ReadyDuration, _ = m.Float64Histogram("messenger.attachment.ready.duration", metric.WithUnit("s"))
|
|
shutdown := func() {
|
|
c, cancel := context.WithTimeout(context.Background(), 4*time.Second)
|
|
defer cancel()
|
|
_ = tp.Shutdown(c)
|
|
_ = mp.Shutdown(c)
|
|
_ = lp.Shutdown(c)
|
|
}
|
|
return log, shutdown, nil
|
|
}
|
|
func Start(ctx context.Context, name string, kind trace.SpanKind) (context.Context, trace.Span) {
|
|
return tracer.Start(ctx, name, trace.WithSpanKind(kind))
|
|
}
|
|
func Failure(span trace.Span, e error) {
|
|
if e != nil {
|
|
span.SetStatus(codes.Error, "dependency operation failed")
|
|
}
|
|
}
|
|
func Inject(ctx context.Context) (string, string) {
|
|
c := propagation.MapCarrier{}
|
|
otel.GetTextMapPropagator().Inject(ctx, c)
|
|
return c.Get("traceparent"), c.Get("tracestate")
|
|
}
|
|
func Extract(ctx context.Context, parent, state string) context.Context {
|
|
return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": parent, "tracestate": state})
|
|
}
|
|
func TraceID(ctx context.Context) string { return trace.SpanContextFromContext(ctx).TraceID().String() }
|
|
func HTTP(ctx context.Context, method, route string, status int, seconds float64) {
|
|
switch method {
|
|
case "GET", "POST", "PUT", "DELETE", "PATCH", "HEAD", "OPTIONS":
|
|
default:
|
|
method = "_OTHER"
|
|
}
|
|
if HTTPCount == nil {
|
|
return
|
|
}
|
|
attrs := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", fmt.Sprintf("%dxx", status/100)))
|
|
HTTPCount.Add(ctx, 1, attrs)
|
|
HTTPDuration.Record(ctx, seconds, attrs)
|
|
}
|
|
func Job(ctx context.Context, outcome string, seconds float64) {
|
|
if Jobs == nil {
|
|
return
|
|
}
|
|
attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome))
|
|
Jobs.Add(ctx, 1, attrs)
|
|
JobDuration.Record(ctx, seconds, attrs)
|
|
}
|
|
func Ready(ctx context.Context, seconds float64) {
|
|
if ReadyDuration != nil {
|
|
ReadyDuration.Record(ctx, seconds)
|
|
}
|
|
}
|
|
func WorkerCapacity(n int) { capacity.Store(int64(n)) }
|
|
func Observe(pool *pgxpool.Pool, worker bool) {
|
|
m := otel.Meter("messenger")
|
|
used, _ := m.Int64ObservableGauge("messenger.db.pool.used")
|
|
limit, _ := m.Int64ObservableGauge("messenger.db.pool.limit")
|
|
waiting, _ := m.Int64ObservableGauge("messenger.db.pool.waiting")
|
|
waits, _ := m.Int64ObservableCounter("messenger.db.pool.waits")
|
|
depth, _ := m.Int64ObservableGauge("messenger.queue.depth")
|
|
age, _ := m.Float64ObservableGauge("messenger.queue.oldest.age", metric.WithUnit("s"))
|
|
busy, _ := m.Int64ObservableGauge("messenger.worker.busy")
|
|
cap, _ := m.Int64ObservableGauge("messenger.worker.capacity")
|
|
sliTotal, _ := m.Int64ObservableCounter("messenger.attachment.sli.total")
|
|
sliBad, _ := m.Int64ObservableCounter("messenger.attachment.sli.bad")
|
|
containerRSS, _ := m.Int64ObservableGauge("container.memory.usage", metric.WithUnit("By"))
|
|
containerCPU, _ := m.Float64ObservableCounter("container.cpu.time", metric.WithUnit("s"))
|
|
rss, _ := m.Int64ObservableGauge("process.memory.usage", metric.WithUnit("By"))
|
|
cpu, _ := m.Float64ObservableCounter("process.cpu.time", metric.WithUnit("s"))
|
|
_, _ = m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
|
|
stat := pool.Stat()
|
|
o.ObserveInt64(used, int64(stat.AcquiredConns()))
|
|
o.ObserveInt64(limit, int64(stat.MaxConns()))
|
|
o.ObserveInt64(waits, stat.EmptyAcquireCount())
|
|
o.ObserveInt64(waiting, PendingAcquires.Load())
|
|
o.ObserveInt64(busy, Busy.Load())
|
|
o.ObserveInt64(cap, capacity.Load())
|
|
mem, cputime := processUsage()
|
|
o.ObserveInt64(rss, mem)
|
|
o.ObserveFloat64(cpu, cputime)
|
|
if raw, e := os.ReadFile("/sys/fs/cgroup/memory.current"); e == nil {
|
|
if bytes, e := strconv.ParseInt(strings.TrimSpace(string(raw)), 10, 64); e == nil {
|
|
o.ObserveInt64(containerRSS, bytes)
|
|
}
|
|
}
|
|
if raw, e := os.ReadFile("/sys/fs/cgroup/cpu.stat"); e == nil {
|
|
for _, line := range strings.Split(string(raw), "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 2 && fields[0] == "usage_usec" {
|
|
if us, e := strconv.ParseFloat(fields[1], 64); e == nil {
|
|
o.ObserveFloat64(containerCPU, us/1e6)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if worker {
|
|
// One bounded query on the worker only avoids duplicating global queue
|
|
// gauges per API. A DB failure omits the observation, never invents zero.
|
|
query, cancel := context.WithTimeout(ctx, 300*time.Millisecond)
|
|
defer cancel()
|
|
var n int64
|
|
var oldest float64
|
|
e := pool.QueryRow(query, `SELECT count(*),COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(created_at)),0)::float8 FROM pipeline_jobs WHERE finished_at IS NULL`).Scan(&n, &oldest)
|
|
if e == nil {
|
|
o.ObserveInt64(depth, n)
|
|
o.ObserveFloat64(age, oldest)
|
|
}
|
|
var matured, bad int64
|
|
e = pool.QueryRow(query, `SELECT count(*),count(*) FILTER (WHERE ready_at IS NULL OR ready_at>created_at+interval '5 seconds') FROM pipeline_jobs WHERE created_at<=clock_timestamp()-interval '5 seconds'`).Scan(&matured, &bad)
|
|
if e == nil {
|
|
o.ObserveInt64(sliTotal, matured)
|
|
o.ObserveInt64(sliBad, bad)
|
|
}
|
|
}
|
|
return nil
|
|
}, used, limit, waits, waiting, depth, age, busy, cap, rss, cpu, sliTotal, sliBad, containerRSS, containerCPU)
|
|
}
|
|
func processUsage() (int64, float64) {
|
|
raw, e := os.ReadFile("/proc/self/stat")
|
|
if e != nil {
|
|
return 0, 0
|
|
}
|
|
end := strings.LastIndex(string(raw), ")")
|
|
if end < 0 {
|
|
return 0, 0
|
|
}
|
|
fields := strings.Fields(string(raw)[end+1:])
|
|
if len(fields) < 22 {
|
|
return 0, 0
|
|
}
|
|
u, _ := strconv.ParseFloat(fields[11], 64)
|
|
s, _ := strconv.ParseFloat(fields[12], 64)
|
|
pages, _ := strconv.ParseInt(fields[21], 10, 64)
|
|
return pages * int64(os.Getpagesize()), (u + s) / 100
|
|
}
|
|
|
|
type contextHandler struct {
|
|
next slog.Handler
|
|
service string
|
|
}
|
|
|
|
func (h *contextHandler) Enabled(ctx context.Context, l slog.Level) bool {
|
|
return h.next.Enabled(ctx, l)
|
|
}
|
|
func (h *contextHandler) Handle(ctx context.Context, r slog.Record) error {
|
|
r.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message))
|
|
sc := trace.SpanContextFromContext(ctx)
|
|
if sc.IsValid() {
|
|
r.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String()))
|
|
}
|
|
return h.next.Handle(ctx, r)
|
|
}
|
|
func (h *contextHandler) WithAttrs(a []slog.Attr) slog.Handler {
|
|
return &contextHandler{next: h.next.WithAttrs(a), service: h.service}
|
|
}
|
|
func (h *contextHandler) WithGroup(g string) slog.Handler {
|
|
return &contextHandler{next: h.next.WithGroup(g), service: h.service}
|
|
}
|
|
|
|
type tee struct{ a, b slog.Handler }
|
|
|
|
func (h *tee) Enabled(ctx context.Context, l slog.Level) bool {
|
|
return h.a.Enabled(ctx, l) || h.b.Enabled(ctx, l)
|
|
}
|
|
func (h *tee) Handle(ctx context.Context, r slog.Record) error {
|
|
_ = h.a.Handle(ctx, r.Clone())
|
|
return h.b.Handle(ctx, r)
|
|
}
|
|
func (h *tee) WithAttrs(a []slog.Attr) slog.Handler { return &tee{h.a.WithAttrs(a), h.b.WithAttrs(a)} }
|
|
func (h *tee) WithGroup(g string) slog.Handler { return &tee{h.a.WithGroup(g), h.b.WithGroup(g)} }
|