274 lines
10 KiB
Go
274 lines
10 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
|
|
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
|
|
otellog "go.opentelemetry.io/otel/log"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdklog "go.opentelemetry.io/otel/sdk/log"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
const telemetryScope = "mtusi/messenger"
|
|
|
|
var traceContext propagation.TraceContext
|
|
|
|
type telemetry struct {
|
|
logger otellog.Logger
|
|
requests metric.Int64Counter
|
|
httpTime metric.Float64Histogram
|
|
jobs metric.Int64Counter
|
|
jobTime metric.Float64Histogram
|
|
readyTime metric.Float64Histogram
|
|
workerBusy atomic.Int64
|
|
}
|
|
|
|
func startTelemetry(role string, db *pgxpool.Pool, poolTrace *databaseTracer) (*telemetry, func(), error) {
|
|
if env("OTEL_SDK_DISABLED", "false") == "true" {
|
|
return nil, func() {}, nil
|
|
}
|
|
ctx := context.Background()
|
|
host, err := os.Hostname()
|
|
if err != nil {
|
|
return nil, nil, errors.New("cannot identify telemetry instance")
|
|
}
|
|
res := resource.NewSchemaless(
|
|
attribute.String("service.name", "messenger-"+role),
|
|
attribute.String("service.version", "6.0.0"),
|
|
attribute.String("service.instance.id", host),
|
|
attribute.String("deployment.environment.name", env("APP_ENV", "lab")),
|
|
)
|
|
const exportTimeout = 2 * time.Second
|
|
tracer, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(exportTimeout),
|
|
otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
|
|
if err != nil {
|
|
return nil, nil, errors.New("invalid OTLP trace configuration")
|
|
}
|
|
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithSampler(sdktrace.AlwaysSample()),
|
|
sdktrace.WithBatcher(tracer, sdktrace.WithMaxQueueSize(1024), sdktrace.WithMaxExportBatchSize(128),
|
|
sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(exportTimeout)))
|
|
var mp *sdkmetric.MeterProvider
|
|
var lp *sdklog.LoggerProvider
|
|
shutdown := func() {
|
|
end, cancel := context.WithTimeout(context.Background(), exportTimeout)
|
|
defer cancel()
|
|
if lp != nil {
|
|
_ = lp.Shutdown(end)
|
|
}
|
|
if mp != nil {
|
|
_ = mp.Shutdown(end)
|
|
}
|
|
_ = tp.Shutdown(end)
|
|
}
|
|
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(exportTimeout),
|
|
otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
|
|
if err != nil {
|
|
shutdown()
|
|
return nil, nil, errors.New("invalid OTLP metric configuration")
|
|
}
|
|
reader := sdkmetric.NewPeriodicReader(metrics, sdkmetric.WithInterval(2*time.Second),
|
|
sdkmetric.WithTimeout(exportTimeout))
|
|
mp = sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(reader))
|
|
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(exportTimeout),
|
|
otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
|
|
if err != nil {
|
|
shutdown()
|
|
return nil, nil, errors.New("invalid OTLP log configuration")
|
|
}
|
|
lp = sdklog.NewLoggerProvider(sdklog.WithResource(res),
|
|
sdklog.WithProcessor(sdklog.NewBatchProcessor(logs, sdklog.WithMaxQueueSize(1024),
|
|
sdklog.WithExportMaxBatchSize(128), sdklog.WithExportInterval(time.Second),
|
|
sdklog.WithExportBufferSize(1), sdklog.WithExportTimeout(exportTimeout))))
|
|
t, err := makeTelemetry(mp.Meter(telemetryScope))
|
|
if err == nil {
|
|
err = observeRuntime(mp.Meter(telemetryScope), role, db, poolTrace)
|
|
}
|
|
if err == nil && role == "worker" {
|
|
err = t.observeWorker(mp.Meter(telemetryScope))
|
|
}
|
|
if err != nil {
|
|
shutdown()
|
|
return nil, nil, fmt.Errorf("register telemetry instruments: %w", err)
|
|
}
|
|
t.logger = lp.Logger(telemetryScope)
|
|
otel.SetTracerProvider(tp)
|
|
// Exporter errors can contain the configured endpoint. Local diagnostics
|
|
// report the fixed event only, and never feed it back into the OTLP queue.
|
|
otel.SetErrorHandler(otel.ErrorHandlerFunc(func(error) { slog.Warn("telemetry export failed") }))
|
|
return t, shutdown, nil
|
|
}
|
|
|
|
func makeTelemetry(m metric.Meter) (*telemetry, error) {
|
|
t := &telemetry{}
|
|
var err, errHTTP, errJobs, errJobTime, errReady error
|
|
t.requests, err = m.Int64Counter("messenger_http_requests")
|
|
t.httpTime, errHTTP = m.Float64Histogram("messenger_http_duration", metric.WithUnit("s"),
|
|
metric.WithExplicitBucketBoundaries(.002, .005, .01, .025, .05, .1, .25, .5, 1, 2, 5, 10, 30))
|
|
t.jobs, errJobs = m.Int64Counter("messenger_job_attempts")
|
|
t.jobTime, errJobTime = m.Float64Histogram("messenger_job_duration", metric.WithUnit("s"),
|
|
metric.WithExplicitBucketBoundaries(.01, .025, .05, .1, .25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120))
|
|
t.readyTime, errReady = m.Float64Histogram("messenger_attachment_time_to_ready", metric.WithUnit("s"),
|
|
metric.WithExplicitBucketBoundaries(.25, .5, 1, 2, 3, 5, 10, 20, 30, 60, 120, 300))
|
|
return t, errors.Join(err, errHTTP, errJobs, errJobTime, errReady)
|
|
}
|
|
|
|
func (t *telemetry) event(ctx context.Context, event string, level otellog.Severity, fields ...otellog.KeyValue) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
var record otellog.Record
|
|
record.SetTimestamp(time.Now())
|
|
record.SetSeverity(level)
|
|
record.SetSeverityText(level.String())
|
|
record.SetBody(otellog.StringValue(event))
|
|
record.AddAttributes(otellog.String("event", event))
|
|
record.AddAttributes(fields...)
|
|
t.logger.Emit(ctx, record)
|
|
}
|
|
|
|
func (t *telemetry) finishHTTP(ctx context.Context, route, method, requestID string, status int, start time.Time) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
class := strconv.Itoa(status/100) + "xx"
|
|
dimensions := metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
|
|
attribute.String("status_class", class))
|
|
duration := time.Since(start).Seconds()
|
|
t.requests.Add(ctx, 1, dimensions)
|
|
if class != "5xx" {
|
|
t.requests.Add(ctx, 0, metric.WithAttributes(attribute.String("method", method), attribute.String("route", route),
|
|
attribute.String("status_class", "5xx")))
|
|
}
|
|
t.httpTime.Record(ctx, duration, dimensions)
|
|
span := trace.SpanFromContext(ctx)
|
|
span.SetAttributes(attribute.Int("http.response.status_code", status))
|
|
severity := otellog.SeverityInfo
|
|
if status >= 500 {
|
|
span.SetStatus(codes.Error, "server_error")
|
|
severity = otellog.SeverityError
|
|
}
|
|
t.event(ctx, "http.request", severity, otellog.String("request_id", requestID),
|
|
otellog.String("route", route), otellog.String("method", method), otellog.Int("status", status),
|
|
otellog.Float64("duration_seconds", duration))
|
|
}
|
|
|
|
func (t *telemetry) finishJob(ctx context.Context, j imageJob, outcome string, started time.Time) {
|
|
if t == nil {
|
|
return
|
|
}
|
|
duration := time.Since(started).Seconds()
|
|
dimensions := metric.WithAttributes(attribute.String("job_type", j.Type), attribute.String("outcome", outcome))
|
|
t.jobs.Add(ctx, 1, dimensions)
|
|
t.jobTime.Record(ctx, duration, dimensions)
|
|
severity := otellog.SeverityInfo
|
|
if outcome != "ready" {
|
|
trace.SpanFromContext(ctx).SetStatus(codes.Error, outcome)
|
|
severity = otellog.SeverityError
|
|
}
|
|
t.event(ctx, "image.attempt", severity, otellog.String("job_id", j.JobID),
|
|
otellog.Int("attempt", j.Attempt), otellog.String("outcome", outcome),
|
|
otellog.Float64("duration_seconds", duration))
|
|
}
|
|
|
|
func safeHTTPRoute(method, path string) (string, string) {
|
|
switch method {
|
|
case "GET", "POST", "DELETE", "PUT", "PATCH", "HEAD", "OPTIONS":
|
|
default:
|
|
method = "OTHER"
|
|
}
|
|
switch path {
|
|
case "/api/v1/users", "/api/v1/users/me", "/api/v1/auth/sessions",
|
|
"/api/v1/auth/sessions/current", "/api/v1/chats":
|
|
return method, path
|
|
}
|
|
parts := strings.SplitN(path, "/", 7)
|
|
if len(parts) >= 5 && parts[0] == "" && parts[1] == "api" && parts[2] == "v1" {
|
|
switch parts[3] {
|
|
case "chats":
|
|
if len(parts) == 5 {
|
|
return method, "/api/v1/chats/{chat_id}"
|
|
}
|
|
if len(parts) == 6 && (parts[5] == "messages" || parts[5] == "attachments") {
|
|
return method, "/api/v1/chats/{chat_id}/" + parts[5]
|
|
}
|
|
case "attachments":
|
|
if len(parts) == 5 {
|
|
return method, "/api/v1/attachments/{attachment_id}"
|
|
}
|
|
if len(parts) == 6 && parts[5] == "content" {
|
|
return method, "/api/v1/attachments/{attachment_id}/content"
|
|
}
|
|
}
|
|
}
|
|
return method, "unmatched"
|
|
}
|
|
|
|
func startSpan(ctx context.Context, name string, opts ...trace.SpanStartOption) (context.Context, trace.Span) {
|
|
return otel.Tracer(telemetryScope).Start(ctx, name, opts...)
|
|
}
|
|
|
|
func endSpan(span trace.Span, err error) {
|
|
if err != nil {
|
|
span.SetStatus(codes.Error, "operation_failed")
|
|
}
|
|
span.End()
|
|
}
|
|
|
|
func jobSpanAttributes(j imageJob) trace.SpanStartOption {
|
|
return trace.WithAttributes(attribute.String("messaging.system", "valkey"),
|
|
attribute.String("messaging.destination.name", imageStream),
|
|
attribute.String("messaging.message.id", j.JobID), attribute.Int("job.attempt", j.Attempt))
|
|
}
|
|
|
|
func (t *telemetry) startJob(ctx context.Context, original propagation.MapCarrier, j imageJob, producer bool) (context.Context, trace.Span) {
|
|
if t != nil {
|
|
ctx = traceContext.Extract(ctx, original)
|
|
}
|
|
if producer {
|
|
return startSpan(ctx, "attachment-jobs publish", trace.WithSpanKind(trace.SpanKindProducer), jobSpanAttributes(j))
|
|
}
|
|
return startSpan(ctx, "attachment-jobs process", trace.WithSpanKind(trace.SpanKindConsumer), jobSpanAttributes(j))
|
|
}
|
|
|
|
// pgx receives full SQL and bind values. Export a fixed span name and status
|
|
// only, and skip calls outside a traced request or consumer.
|
|
func (*databaseTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
|
|
if !trace.SpanFromContext(ctx).IsRecording() {
|
|
return ctx
|
|
}
|
|
child, span := startSpan(ctx, "postgresql query", trace.WithSpanKind(trace.SpanKindClient),
|
|
trace.WithAttributes(attribute.String("db.system.name", "postgresql")))
|
|
return context.WithValue(child, querySpanKey{}, span)
|
|
}
|
|
|
|
type querySpanKey struct{}
|
|
|
|
func (*databaseTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
|
if span, ok := ctx.Value(querySpanKey{}).(trace.Span); ok {
|
|
if errors.Is(data.Err, pgx.ErrNoRows) {
|
|
data.Err = nil
|
|
}
|
|
endSpan(span, data.Err)
|
|
}
|
|
}
|