feat(storage): lab6 add PostgreSQL, Redis and S3 tracing and DB pool metrics

This commit is contained in:
Голиков Maximillian Антон
2026-11-16 14:48:07 +03:00
parent c403b23595
commit 472b4761d6
11 changed files with 225 additions and 3 deletions
+7
View File
@@ -72,6 +72,13 @@ func main() {
}
defer pool.Close()
registration, err := postgres.ObservePool(pool)
if err != nil {
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
} else {
defer registration.Unregister()
}
store := postgres.New(pool)
objectStorage, err := s3store.New(ctx, s3store.Config{
+7
View File
@@ -81,6 +81,13 @@ func main() {
}
defer pool.Close()
registration, err := postgres.ObservePool(pool)
if err != nil {
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
} else {
defer registration.Unregister()
}
store := postgres.New(pool)
rabbitPub, err := broker.NewRabbitPublisher(cfg.RabbitMQURL)
+7
View File
@@ -56,6 +56,13 @@ func main() {
}
defer pool.Close()
registration, err := postgres.ObservePool(pool)
if err != nil {
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
} else {
defer registration.Unregister()
}
store := postgres.New(pool)
objectStorage, err := s3store.New(ctx, s3store.Config{
Endpoint: cfg.S3Endpoint,
+2
View File
@@ -25,6 +25,7 @@ require (
require (
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/ebitengine/purego v0.10.1 // indirect
github.com/felixge/httpsnoop v1.1.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
@@ -82,6 +83,7 @@ require (
github.com/zeebo/xxh3 v1.1.0 // indirect
go.mongodb.org/mongo-driver/v2 v2.9.1 // indirect
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
+4
View File
@@ -31,6 +31,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw=
github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
@@ -179,6 +181,8 @@ go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNR
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4=
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0 h1:Vgm7vrC26vvecffpDcy2UINqBvASUg5qkfZle3Yrkug=
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0/go.mod h1:zgCChxie/dA3qG7IgZGswax6voxkOR4+33msLT/RgNg=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE=
+6
View File
@@ -47,6 +47,12 @@ func Connect(ctx context.Context, cfg Config) (*pgxpool.Pool, error) {
}
poolCfg.HealthCheckPeriod = 10 * time.Second
tracer, err := newPoolTracer()
if err != nil {
return nil, fmt.Errorf("метрики пула: %w", err)
}
poolCfg.ConnConfig.Tracer = tracer
pool, err := pgxpool.NewWithConfig(ctx, poolCfg)
if err != nil {
return nil, fmt.Errorf("создание пула: %w", err)
@@ -0,0 +1,99 @@
package postgres
import (
"context"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"high-loud-apps-labs/internal/telemetry"
)
type poolTracer struct {
tracer trace.Tracer
duration metric.Float64Histogram
waiting metric.Int64UpDownCounter
}
func newPoolTracer() (*poolTracer, error) {
meter := telemetry.Meter("storage.postgres")
duration, err := meter.Float64Histogram("db_pool_acquire_duration_seconds",
metric.WithUnit("s"),
metric.WithExplicitBucketBoundaries(0.0001, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5),
)
if err != nil {
return nil, err
}
waiting, err := meter.Int64UpDownCounter("db_pool_waiting",
metric.WithDescription("Число незавершённых вызовов Acquire"))
if err != nil {
return nil, err
}
return &poolTracer{tracer: telemetry.Tracer("storage.postgres"), duration: duration, waiting: waiting}, nil
}
type acquireStartKey struct{}
func (t *poolTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
ctx, _ = t.tracer.Start(ctx, "postgresql.acquire")
t.waiting.Add(ctx, 1)
return context.WithValue(ctx, acquireStartKey{}, time.Now())
}
func (t *poolTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
t.waiting.Add(ctx, -1)
t.duration.Record(ctx, time.Since(ctx.Value(acquireStartKey{}).(time.Time)).Seconds())
endSpan(ctx, data.Err)
}
func (t *poolTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
// Передаём только известный SQL-глагол, без текста запроса и параметров.
op := "QUERY"
sql := strings.TrimSpace(data.SQL)
if i := strings.IndexAny(sql, " \r\n\t;("); i >= 0 {
sql = sql[:i]
}
switch verb := strings.ToUpper(sql); verb {
case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK", "WITH":
op = verb
}
ctx, _ = t.tracer.Start(ctx, "postgresql."+op, trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op)))
return ctx
}
func (t *poolTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
endSpan(ctx, data.Err)
}
func endSpan(ctx context.Context, err error) {
span := trace.SpanFromContext(ctx)
if err != nil {
span.SetStatus(codes.Error, "операция PostgreSQL не выполнена")
}
span.End()
}
func ObservePool(pool *pgxpool.Pool) (metric.Registration, error) {
meter := telemetry.Meter("storage.postgres")
connections, err := meter.Int64ObservableGauge("db_pool_connections")
if err != nil {
return nil, err
}
return meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
stat := pool.Stat()
observe := func(state string, count int32) {
observer.ObserveInt64(connections, int64(count), metric.WithAttributes(attribute.String("state", state)))
}
observe("used", stat.AcquiredConns())
observe("idle", stat.IdleConns())
observe("max", stat.MaxConns())
return nil
}, connections)
}
+3
View File
@@ -6,6 +6,8 @@ import (
"time"
"github.com/redis/go-redis/v9"
"high-loud-apps-labs/internal/telemetry"
)
func NewClient(url string) (*redis.Client, error) {
@@ -22,6 +24,7 @@ func NewClient(url string) (*redis.Client, error) {
opt.ConnMaxIdleTime = time.Minute
client := redis.NewClient(opt)
client.AddHook(tracingHook{tracer: telemetry.Tracer("storage.redis")})
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
+38
View File
@@ -0,0 +1,38 @@
package redis
import (
"context"
"errors"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
type tracingHook struct{ tracer trace.Tracer }
func (h tracingHook) DialHook(next redis.DialHook) redis.DialHook { return next }
func (h tracingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
return func(ctx context.Context, cmd redis.Cmder) error {
return h.run(ctx, cmd.Name(), func(ctx context.Context) error { return next(ctx, cmd) })
}
}
func (h tracingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
return func(ctx context.Context, cmds []redis.Cmder) error {
return h.run(ctx, "pipeline", func(ctx context.Context) error { return next(ctx, cmds) })
}
}
func (h tracingHook) run(ctx context.Context, op string, call func(context.Context) error) error {
ctx, span := h.tracer.Start(ctx, "redis."+op, trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("db.system.name", "redis"), attribute.String("db.operation.name", op)))
defer span.End()
err := call(ctx)
if err != nil && !errors.Is(err, redis.Nil) {
span.SetStatus(codes.Error, "операция Redis не выполнена")
}
return err
}
+20 -3
View File
@@ -3,9 +3,11 @@ package s3
import (
"context"
"io"
"net/http"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"high-loud-apps-labs/internal/domain"
)
@@ -25,10 +27,25 @@ type MinioStorage struct {
}
func New(ctx context.Context, cfg Config) (*MinioStorage, error) {
base, err := minio.DefaultTransport(cfg.UseSSL)
if err != nil {
return nil, err
}
// Оборачиваем базовый транспорт в otelhttp.
// Опция WithSpanNameFormatter задаёт красивое имя спана (s3.PUT, s3.GET),
// чтобы в Grafana не смотреть на дефолтное "HTTP PUT".
tracedTransport := otelhttp.NewTransport(base,
otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
return "s3." + r.Method
}),
)
client, err := minio.New(cfg.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: cfg.UseSSL,
Region: cfg.Region,
Transport: tracedTransport,
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
Secure: cfg.UseSSL,
Region: cfg.Region,
})
if err != nil {
return nil, err
+32
View File
@@ -0,0 +1,32 @@
package s3
import (
"net/http"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
)
// Один span на HTTP-попытку MinIO до получения заголовков ответа.
type tracingTransport struct {
next http.RoundTripper
tracer trace.Tracer
}
func (t tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
ctx, span := t.tracer.Start(req.Context(), "s3."+req.Method, trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("http.request.method", req.Method)))
defer span.End()
resp, err := t.next.RoundTrip(req.WithContext(ctx))
if err != nil {
span.SetStatus(codes.Error, "ошибка соединения с S3")
} else {
span.SetAttributes(attribute.Int("http.response.status_code", resp.StatusCode))
if resp.StatusCode >= 400 {
span.SetStatus(codes.Error, http.StatusText(resp.StatusCode))
}
}
return resp, err
}