diff --git a/backend/cmd/reconciler/main.go b/backend/cmd/reconciler/main.go index a7c11c2..a9bc3b6 100644 --- a/backend/cmd/reconciler/main.go +++ b/backend/cmd/reconciler/main.go @@ -72,6 +72,13 @@ func main() { } defer pool.Close() + registration, err := postgres.ObservePool(pool) + if err != nil { + slog.Warn("Не удалось зарегистрировать метрики пула", "error", err) + } else { + defer registration.Unregister() + } + store := postgres.New(pool) objectStorage, err := s3store.New(ctx, s3store.Config{ diff --git a/backend/cmd/server/main.go b/backend/cmd/server/main.go index 08f68e5..f3a2d5b 100644 --- a/backend/cmd/server/main.go +++ b/backend/cmd/server/main.go @@ -81,6 +81,13 @@ func main() { } defer pool.Close() + registration, err := postgres.ObservePool(pool) + if err != nil { + slog.Warn("Не удалось зарегистрировать метрики пула", "error", err) + } else { + defer registration.Unregister() + } + store := postgres.New(pool) rabbitPub, err := broker.NewRabbitPublisher(cfg.RabbitMQURL) diff --git a/backend/cmd/worker/main.go b/backend/cmd/worker/main.go index c896f4a..69f6373 100644 --- a/backend/cmd/worker/main.go +++ b/backend/cmd/worker/main.go @@ -56,6 +56,13 @@ func main() { } defer pool.Close() + registration, err := postgres.ObservePool(pool) + if err != nil { + slog.Warn("Не удалось зарегистрировать метрики пула", "error", err) + } else { + defer registration.Unregister() + } + store := postgres.New(pool) objectStorage, err := s3store.New(ctx, s3store.Config{ Endpoint: cfg.S3Endpoint, diff --git a/backend/go.mod b/backend/go.mod index 9b66899..dc29ad2 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -25,6 +25,7 @@ require ( require ( github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect @@ -82,6 +83,7 @@ require ( github.com/zeebo/xxh3 v1.1.0 // indirect go.mongodb.org/mongo-driver/v2 v2.9.1 // indirect go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect diff --git a/backend/go.sum b/backend/go.sum index 81076f0..8c58cb3 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -31,6 +31,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI= @@ -179,6 +181,8 @@ go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNR go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0 h1:Vgm7vrC26vvecffpDcy2UINqBvASUg5qkfZle3Yrkug= go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0/go.mod h1:zgCChxie/dA3qG7IgZGswax6voxkOR4+33msLT/RgNg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw= go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM= go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU= go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE= diff --git a/backend/internal/storage/postgres/db.go b/backend/internal/storage/postgres/db.go index ad27e83..04ac5d4 100644 --- a/backend/internal/storage/postgres/db.go +++ b/backend/internal/storage/postgres/db.go @@ -47,6 +47,12 @@ func Connect(ctx context.Context, cfg Config) (*pgxpool.Pool, error) { } poolCfg.HealthCheckPeriod = 10 * time.Second + tracer, err := newPoolTracer() + if err != nil { + return nil, fmt.Errorf("метрики пула: %w", err) + } + poolCfg.ConnConfig.Tracer = tracer + pool, err := pgxpool.NewWithConfig(ctx, poolCfg) if err != nil { return nil, fmt.Errorf("создание пула: %w", err) diff --git a/backend/internal/storage/postgres/tracer.go b/backend/internal/storage/postgres/tracer.go new file mode 100644 index 0000000..fe652cf --- /dev/null +++ b/backend/internal/storage/postgres/tracer.go @@ -0,0 +1,99 @@ +package postgres + +import ( + "context" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "high-loud-apps-labs/internal/telemetry" +) + +type poolTracer struct { + tracer trace.Tracer + duration metric.Float64Histogram + waiting metric.Int64UpDownCounter +} + +func newPoolTracer() (*poolTracer, error) { + meter := telemetry.Meter("storage.postgres") + duration, err := meter.Float64Histogram("db_pool_acquire_duration_seconds", + metric.WithUnit("s"), + metric.WithExplicitBucketBoundaries(0.0001, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5), + ) + if err != nil { + return nil, err + } + waiting, err := meter.Int64UpDownCounter("db_pool_waiting", + metric.WithDescription("Число незавершённых вызовов Acquire")) + if err != nil { + return nil, err + } + return &poolTracer{tracer: telemetry.Tracer("storage.postgres"), duration: duration, waiting: waiting}, nil +} + +type acquireStartKey struct{} + +func (t *poolTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context { + ctx, _ = t.tracer.Start(ctx, "postgresql.acquire") + t.waiting.Add(ctx, 1) + return context.WithValue(ctx, acquireStartKey{}, time.Now()) +} + +func (t *poolTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) { + t.waiting.Add(ctx, -1) + t.duration.Record(ctx, time.Since(ctx.Value(acquireStartKey{}).(time.Time)).Seconds()) + endSpan(ctx, data.Err) +} + +func (t *poolTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + // Передаём только известный SQL-глагол, без текста запроса и параметров. + op := "QUERY" + sql := strings.TrimSpace(data.SQL) + if i := strings.IndexAny(sql, " \r\n\t;("); i >= 0 { + sql = sql[:i] + } + switch verb := strings.ToUpper(sql); verb { + case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK", "WITH": + op = verb + } + ctx, _ = t.tracer.Start(ctx, "postgresql."+op, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op))) + return ctx +} + +func (t *poolTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + endSpan(ctx, data.Err) +} + +func endSpan(ctx context.Context, err error) { + span := trace.SpanFromContext(ctx) + if err != nil { + span.SetStatus(codes.Error, "операция PostgreSQL не выполнена") + } + span.End() +} + +func ObservePool(pool *pgxpool.Pool) (metric.Registration, error) { + meter := telemetry.Meter("storage.postgres") + connections, err := meter.Int64ObservableGauge("db_pool_connections") + if err != nil { + return nil, err + } + return meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error { + stat := pool.Stat() + observe := func(state string, count int32) { + observer.ObserveInt64(connections, int64(count), metric.WithAttributes(attribute.String("state", state))) + } + observe("used", stat.AcquiredConns()) + observe("idle", stat.IdleConns()) + observe("max", stat.MaxConns()) + return nil + }, connections) +} diff --git a/backend/internal/storage/redis/client.go b/backend/internal/storage/redis/client.go index 0da6614..fa8e83f 100644 --- a/backend/internal/storage/redis/client.go +++ b/backend/internal/storage/redis/client.go @@ -6,6 +6,8 @@ import ( "time" "github.com/redis/go-redis/v9" + + "high-loud-apps-labs/internal/telemetry" ) func NewClient(url string) (*redis.Client, error) { @@ -22,6 +24,7 @@ func NewClient(url string) (*redis.Client, error) { opt.ConnMaxIdleTime = time.Minute client := redis.NewClient(opt) + client.AddHook(tracingHook{tracer: telemetry.Tracer("storage.redis")}) ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() diff --git a/backend/internal/storage/redis/hook.go b/backend/internal/storage/redis/hook.go new file mode 100644 index 0000000..b352c0c --- /dev/null +++ b/backend/internal/storage/redis/hook.go @@ -0,0 +1,38 @@ +package redis + +import ( + "context" + "errors" + + "github.com/redis/go-redis/v9" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +type tracingHook struct{ tracer trace.Tracer } + +func (h tracingHook) DialHook(next redis.DialHook) redis.DialHook { return next } + +func (h tracingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { + return h.run(ctx, cmd.Name(), func(ctx context.Context) error { return next(ctx, cmd) }) + } +} + +func (h tracingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return func(ctx context.Context, cmds []redis.Cmder) error { + return h.run(ctx, "pipeline", func(ctx context.Context) error { return next(ctx, cmds) }) + } +} + +func (h tracingHook) run(ctx context.Context, op string, call func(context.Context) error) error { + ctx, span := h.tracer.Start(ctx, "redis."+op, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("db.system.name", "redis"), attribute.String("db.operation.name", op))) + defer span.End() + err := call(ctx) + if err != nil && !errors.Is(err, redis.Nil) { + span.SetStatus(codes.Error, "операция Redis не выполнена") + } + return err +} diff --git a/backend/internal/storage/s3/client.go b/backend/internal/storage/s3/client.go index 75cbf95..2cdcbd2 100644 --- a/backend/internal/storage/s3/client.go +++ b/backend/internal/storage/s3/client.go @@ -3,9 +3,11 @@ package s3 import ( "context" "io" + "net/http" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" + "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "high-loud-apps-labs/internal/domain" ) @@ -25,10 +27,25 @@ type MinioStorage struct { } func New(ctx context.Context, cfg Config) (*MinioStorage, error) { + base, err := minio.DefaultTransport(cfg.UseSSL) + if err != nil { + return nil, err + } + + // Оборачиваем базовый транспорт в otelhttp. + // Опция WithSpanNameFormatter задаёт красивое имя спана (s3.PUT, s3.GET), + // чтобы в Grafana не смотреть на дефолтное "HTTP PUT". + tracedTransport := otelhttp.NewTransport(base, + otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string { + return "s3." + r.Method + }), + ) + client, err := minio.New(cfg.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), - Secure: cfg.UseSSL, - Region: cfg.Region, + Transport: tracedTransport, + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.UseSSL, + Region: cfg.Region, }) if err != nil { return nil, err diff --git a/backend/internal/storage/s3/transport.go b/backend/internal/storage/s3/transport.go new file mode 100644 index 0000000..64fc2ec --- /dev/null +++ b/backend/internal/storage/s3/transport.go @@ -0,0 +1,32 @@ +package s3 + +import ( + "net/http" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +// Один span на HTTP-попытку MinIO до получения заголовков ответа. +type tracingTransport struct { + next http.RoundTripper + tracer trace.Tracer +} + +func (t tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + ctx, span := t.tracer.Start(req.Context(), "s3."+req.Method, trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attribute.String("http.request.method", req.Method))) + defer span.End() + + resp, err := t.next.RoundTrip(req.WithContext(ctx)) + if err != nil { + span.SetStatus(codes.Error, "ошибка соединения с S3") + } else { + span.SetAttributes(attribute.Int("http.response.status_code", resp.StatusCode)) + if resp.StatusCode >= 400 { + span.SetStatus(codes.Error, http.StatusText(resp.StatusCode)) + } + } + return resp, err +}