forked from mtusi2026/mtusi2026lab6
353 lines
12 KiB
Go
353 lines
12 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.opentelemetry.io/contrib/bridges/otelslog"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdklog "go.opentelemetry.io/otel/sdk/log"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
const appVersion = "1.6.0"
|
|
|
|
var (
|
|
tracer = otel.Tracer("messenger")
|
|
meter = otel.Meter("messenger")
|
|
|
|
httpRequests = mustCounter("http.server.requests", "{request}", "HTTP requests by method, route template and status class")
|
|
httpDuration = mustHistogram("http.server.request.duration", "s", "HTTP server request duration",
|
|
0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
|
|
jobsProcessed = mustCounter("jobs.processed", "{job}", "Processed job attempts by outcome")
|
|
jobDuration = mustHistogram("job.processing.duration", "s", "Job processing duration",
|
|
0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
|
|
jobTimeToReady = mustHistogram("job.time_to_ready", "s", "Time from durable acceptance to ready",
|
|
0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300)
|
|
|
|
exportErrors atomic.Int64
|
|
)
|
|
|
|
type exportErrorHandler struct{}
|
|
|
|
func (exportErrorHandler) Handle(error) { exportErrors.Add(1) }
|
|
|
|
func mustCounter(name, unit, desc string) metric.Int64Counter {
|
|
c, err := meter.Int64Counter(name, metric.WithUnit(unit), metric.WithDescription(desc))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return c
|
|
}
|
|
|
|
func mustHistogram(name, unit, desc string, buckets ...float64) metric.Float64Histogram {
|
|
h, err := meter.Float64Histogram(name, metric.WithUnit(unit), metric.WithDescription(desc),
|
|
metric.WithExplicitBucketBoundaries(buckets...))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return h
|
|
}
|
|
|
|
type fanoutHandler struct {
|
|
stdout, remote slog.Handler
|
|
service string
|
|
}
|
|
|
|
func (h fanoutHandler) Enabled(ctx context.Context, l slog.Level) bool {
|
|
return h.stdout.Enabled(ctx, l)
|
|
}
|
|
|
|
func (h fanoutHandler) Handle(ctx context.Context, r slog.Record) error {
|
|
local := r.Clone()
|
|
local.AddAttrs(slog.String("service.name", h.service), slog.String("event", r.Message))
|
|
if sc := trace.SpanContextFromContext(ctx); sc.IsValid() {
|
|
local.AddAttrs(slog.String("trace_id", sc.TraceID().String()), slog.String("span_id", sc.SpanID().String()))
|
|
}
|
|
err := h.stdout.Handle(ctx, local)
|
|
remote := r.Clone()
|
|
remote.AddAttrs(slog.String("event", r.Message))
|
|
_ = h.remote.Handle(ctx, remote)
|
|
return err
|
|
}
|
|
|
|
func (h fanoutHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
|
|
return fanoutHandler{stdout: h.stdout.WithAttrs(attrs), remote: h.remote.WithAttrs(attrs), service: h.service}
|
|
}
|
|
|
|
func (h fanoutHandler) WithGroup(name string) slog.Handler {
|
|
return fanoutHandler{stdout: h.stdout.WithGroup(name), remote: h.remote.WithGroup(name), service: h.service}
|
|
}
|
|
|
|
func setupTelemetry(ctx context.Context, defaultService string) (*slog.Logger, func(context.Context)) {
|
|
service := envOr("OTEL_SERVICE_NAME", defaultService)
|
|
res, err := resource.New(ctx,
|
|
resource.WithAttributes(
|
|
attribute.String("service.name", service),
|
|
attribute.String("service.version", appVersion),
|
|
attribute.String("service.instance.id", instanceID),
|
|
attribute.String("deployment.environment.name", envOr("APP_ENV", "lab")),
|
|
),
|
|
resource.WithFromEnv(),
|
|
)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
otel.SetErrorHandler(exportErrorHandler{})
|
|
otel.SetTextMapPropagator(propagation.TraceContext{})
|
|
|
|
traceExp, err := otlptracehttp.New(ctx,
|
|
otlptracehttp.WithTimeout(3*time.Second),
|
|
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
tp := sdktrace.NewTracerProvider(
|
|
sdktrace.WithResource(res),
|
|
sdktrace.WithBatcher(traceExp,
|
|
sdktrace.WithBatchTimeout(time.Second),
|
|
sdktrace.WithMaxQueueSize(2048),
|
|
sdktrace.WithMaxExportBatchSize(512),
|
|
sdktrace.WithExportTimeout(5*time.Second)),
|
|
)
|
|
otel.SetTracerProvider(tp)
|
|
|
|
metricExp, err := otlpmetrichttp.New(ctx,
|
|
otlpmetrichttp.WithTimeout(3*time.Second),
|
|
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
mp := sdkmetric.NewMeterProvider(
|
|
sdkmetric.WithResource(res),
|
|
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(metricExp, sdkmetric.WithInterval(10*time.Second), sdkmetric.WithTimeout(5*time.Second))),
|
|
)
|
|
otel.SetMeterProvider(mp)
|
|
|
|
logExp, err := otlploghttp.New(ctx,
|
|
otlploghttp.WithTimeout(3*time.Second),
|
|
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: true, InitialInterval: time.Second, MaxInterval: 4 * time.Second, MaxElapsedTime: 8 * time.Second}))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
lp := sdklog.NewLoggerProvider(
|
|
sdklog.WithResource(res),
|
|
sdklog.WithProcessor(sdklog.NewBatchProcessor(logExp,
|
|
sdklog.WithMaxQueueSize(2048),
|
|
sdklog.WithExportInterval(time.Second),
|
|
sdklog.WithExportTimeout(5*time.Second))),
|
|
)
|
|
|
|
registerProcessMetrics()
|
|
log := slog.New(fanoutHandler{
|
|
stdout: slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo}),
|
|
remote: otelslog.NewHandler("messenger", otelslog.WithLoggerProvider(lp)),
|
|
service: service,
|
|
})
|
|
shutdown := func(ctx context.Context) {
|
|
_ = tp.Shutdown(ctx)
|
|
_ = mp.Shutdown(ctx)
|
|
_ = lp.Shutdown(ctx)
|
|
}
|
|
return log, shutdown
|
|
}
|
|
|
|
func observableGauge(name, unit, desc string, cb func(context.Context, metric.Int64Observer) error) {
|
|
_, err := meter.Int64ObservableGauge(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithInt64Callback(cb))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func observableCounter(name, unit, desc string, cb func(context.Context, metric.Float64Observer) error) {
|
|
_, err := meter.Float64ObservableCounter(name, metric.WithUnit(unit), metric.WithDescription(desc), metric.WithFloat64Callback(cb))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func registerProcessMetrics() {
|
|
observableCounter("process.cpu.time", "s", "Process CPU time", func(_ context.Context, o metric.Float64Observer) error {
|
|
var ru syscall.Rusage
|
|
if err := syscall.Getrusage(syscall.RUSAGE_SELF, &ru); err != nil {
|
|
return nil
|
|
}
|
|
sec := func(t syscall.Timeval) float64 { return float64(t.Sec) + float64(t.Usec)/1e6 }
|
|
o.Observe(sec(ru.Utime) + sec(ru.Stime))
|
|
return nil
|
|
})
|
|
observableGauge("process.memory.usage", "By", "Resident set size", func(_ context.Context, o metric.Int64Observer) error {
|
|
data, err := os.ReadFile("/proc/self/status")
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
if strings.HasPrefix(line, "VmRSS:") {
|
|
f := strings.Fields(line)
|
|
if len(f) >= 2 {
|
|
var kb int64
|
|
for _, c := range f[1] {
|
|
kb = kb*10 + int64(c-'0')
|
|
}
|
|
o.Observe(kb * 1024)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
observableCounter("telemetry.export.errors", "{error}", "Failed telemetry exports", func(_ context.Context, o metric.Float64Observer) error {
|
|
o.Observe(float64(exportErrors.Load()))
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func registerPoolMetrics(db *pgxpool.Pool) {
|
|
observableGauge("db.pool.connections", "{connection}", "DB pool connections by state", func(_ context.Context, o metric.Int64Observer) error {
|
|
s := db.Stat()
|
|
o.Observe(int64(s.AcquiredConns()), metric.WithAttributes(attribute.String("state", "used")))
|
|
o.Observe(int64(s.IdleConns()), metric.WithAttributes(attribute.String("state", "idle")))
|
|
return nil
|
|
})
|
|
observableGauge("db.pool.max_connections", "{connection}", "DB pool size limit", func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(int64(db.Stat().MaxConns()))
|
|
return nil
|
|
})
|
|
observableCounter("db.pool.empty_acquires", "{acquire}", "Acquires that had to wait for a connection", func(_ context.Context, o metric.Float64Observer) error {
|
|
o.Observe(float64(db.Stat().EmptyAcquireCount()))
|
|
return nil
|
|
})
|
|
observableCounter("db.pool.acquire.wait", "s", "Total time spent acquiring connections", func(_ context.Context, o metric.Float64Observer) error {
|
|
o.Observe(db.Stat().AcquireDuration().Seconds())
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func registerQueueMetrics(db *pgxpool.Pool) {
|
|
var mu sync.Mutex
|
|
var depth int64
|
|
var age float64
|
|
refresh := func(ctx context.Context) {
|
|
ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
|
defer cancel()
|
|
var d int64
|
|
var a float64
|
|
err := db.QueryRow(ctx, `
|
|
select count(*),
|
|
coalesce(extract(epoch from now() - min(next_attempt_at) filter (where next_attempt_at <= now())), 0)
|
|
from jobs where status = 'queued'`).Scan(&d, &a)
|
|
if err != nil {
|
|
return
|
|
}
|
|
mu.Lock()
|
|
depth, age = d, a
|
|
mu.Unlock()
|
|
}
|
|
attrs := metric.WithAttributes(attribute.String("queue", jobStream))
|
|
observableGauge("queue.depth", "{job}", "Queued jobs", func(ctx context.Context, o metric.Int64Observer) error {
|
|
refresh(ctx)
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
o.Observe(depth, attrs)
|
|
return nil
|
|
})
|
|
_, err := meter.Float64ObservableGauge("queue.oldest_job_age", metric.WithUnit("s"), metric.WithDescription("Age of the oldest available queued job"),
|
|
metric.WithFloat64Callback(func(_ context.Context, o metric.Float64Observer) error {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
o.Observe(age, attrs)
|
|
return nil
|
|
}))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
}
|
|
|
|
func registerWorkerMetrics(w *Worker, concurrency int) {
|
|
observableGauge("worker.busy", "{slot}", "Jobs being processed", func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(w.busy.Load())
|
|
return nil
|
|
})
|
|
observableGauge("worker.capacity", "{slot}", "Configured worker concurrency", func(_ context.Context, o metric.Int64Observer) error {
|
|
o.Observe(int64(concurrency))
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func childSpan(ctx context.Context, name string, kind trace.SpanKind, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
|
|
if !trace.SpanContextFromContext(ctx).IsValid() {
|
|
return ctx, trace.SpanFromContext(ctx)
|
|
}
|
|
return tracer.Start(ctx, name, trace.WithSpanKind(kind), trace.WithAttributes(attrs...))
|
|
}
|
|
|
|
func endSpan(sp trace.Span, err error) {
|
|
if err != nil {
|
|
sp.RecordError(err)
|
|
sp.SetStatus(codes.Error, err.Error())
|
|
}
|
|
sp.End()
|
|
}
|
|
|
|
type traceCarrier struct{ traceparent, tracestate string }
|
|
|
|
func injectTrace(ctx context.Context) traceCarrier {
|
|
m := propagation.MapCarrier{}
|
|
otel.GetTextMapPropagator().Inject(ctx, m)
|
|
return traceCarrier{traceparent: m.Get("traceparent"), tracestate: m.Get("tracestate")}
|
|
}
|
|
|
|
func extractTrace(ctx context.Context, c traceCarrier) context.Context {
|
|
if c.traceparent == "" {
|
|
return ctx
|
|
}
|
|
m := propagation.MapCarrier{"traceparent": c.traceparent}
|
|
if c.tracestate != "" {
|
|
m["tracestate"] = c.tracestate
|
|
}
|
|
return otel.GetTextMapPropagator().Extract(ctx, m)
|
|
}
|
|
|
|
type dbTracer struct{}
|
|
|
|
type dbSpanKey struct{}
|
|
|
|
func (dbTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
|
|
if !trace.SpanContextFromContext(ctx).IsValid() {
|
|
return ctx
|
|
}
|
|
op := "query"
|
|
if f := strings.Fields(data.SQL); len(f) > 0 {
|
|
op = strings.ToUpper(f[0])
|
|
}
|
|
ctx, sp := tracer.Start(ctx, "db "+op, trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes(
|
|
attribute.String("db.system.name", "postgresql"),
|
|
attribute.String("db.operation.name", op),
|
|
attribute.String("db.query.text", data.SQL),
|
|
))
|
|
return context.WithValue(ctx, dbSpanKey{}, sp)
|
|
}
|
|
|
|
func (dbTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
|
if sp, ok := ctx.Value(dbSpanKey{}).(trace.Span); ok {
|
|
endSpan(sp, data.Err)
|
|
}
|
|
}
|