forked from mtusi2026/mtusi2026lab6
feat(storage): lab6 add PostgreSQL, Redis and S3 tracing and DB pool metrics
This commit is contained in:
@@ -72,6 +72,13 @@ func main() {
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
registration, err := postgres.ObservePool(pool)
|
||||
if err != nil {
|
||||
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
|
||||
} else {
|
||||
defer registration.Unregister()
|
||||
}
|
||||
|
||||
store := postgres.New(pool)
|
||||
|
||||
objectStorage, err := s3store.New(ctx, s3store.Config{
|
||||
|
||||
@@ -81,6 +81,13 @@ func main() {
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
registration, err := postgres.ObservePool(pool)
|
||||
if err != nil {
|
||||
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
|
||||
} else {
|
||||
defer registration.Unregister()
|
||||
}
|
||||
|
||||
store := postgres.New(pool)
|
||||
|
||||
rabbitPub, err := broker.NewRabbitPublisher(cfg.RabbitMQURL)
|
||||
|
||||
@@ -56,6 +56,13 @@ func main() {
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
registration, err := postgres.ObservePool(pool)
|
||||
if err != nil {
|
||||
slog.Warn("Не удалось зарегистрировать метрики пула", "error", err)
|
||||
} else {
|
||||
defer registration.Unregister()
|
||||
}
|
||||
|
||||
store := postgres.New(pool)
|
||||
objectStorage, err := s3store.New(ctx, s3store.Config{
|
||||
Endpoint: cfg.S3Endpoint,
|
||||
|
||||
@@ -25,6 +25,7 @@ require (
|
||||
require (
|
||||
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/felixge/httpsnoop v1.1.0 // indirect
|
||||
github.com/go-logr/logr v1.4.4 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
|
||||
@@ -82,6 +83,7 @@ require (
|
||||
github.com/zeebo/xxh3 v1.1.0 // indirect
|
||||
go.mongodb.org/mongo-driver/v2 v2.9.1 // indirect
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
|
||||
@@ -31,6 +31,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc=
|
||||
github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE=
|
||||
github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw=
|
||||
github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
|
||||
github.com/gabriel-vasile/mimetype v1.4.15 h1:05iP/CYtZ/w455R/KZM6rZ5ieAdh99UPtd+d3YzLmaI=
|
||||
@@ -179,6 +181,8 @@ go.opentelemetry.io/contrib/bridges/otelslog v0.21.0 h1:/CdgE52i1OEPHhsGUqrggYNR
|
||||
go.opentelemetry.io/contrib/bridges/otelslog v0.21.0/go.mod h1:9pBdbVbuehIDzxN4accnzzxEUPT8Nq7grbZMo+J1VN4=
|
||||
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0 h1:Vgm7vrC26vvecffpDcy2UINqBvASUg5qkfZle3Yrkug=
|
||||
go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin v0.72.0/go.mod h1:zgCChxie/dA3qG7IgZGswax6voxkOR4+33msLT/RgNg=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0 h1:LxwW/9ctSCv+QkE/cLR7M91ZIkXNMqJtEMi1vCw9U8s=
|
||||
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.72.0/go.mod h1:tOsftB4SslBwwErVEPaenU2RpThXWPIU8DoJHEC4dyw=
|
||||
go.opentelemetry.io/otel v1.47.0 h1:j7ALJ/zgkS7Z6aeJW09p8VC9804bC+PpeTfCD4XPnOM=
|
||||
go.opentelemetry.io/otel v1.47.0/go.mod h1:8wS9O2qfXrYrzp6hIF/HOYJJf/wIhFPhR2xLuP+iXQU=
|
||||
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 h1:5WVp40k2fJHuc5BtNTgKk+EugKFT8Eg3lywFfCPI/nE=
|
||||
|
||||
@@ -47,6 +47,12 @@ func Connect(ctx context.Context, cfg Config) (*pgxpool.Pool, error) {
|
||||
}
|
||||
poolCfg.HealthCheckPeriod = 10 * time.Second
|
||||
|
||||
tracer, err := newPoolTracer()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("метрики пула: %w", err)
|
||||
}
|
||||
poolCfg.ConnConfig.Tracer = tracer
|
||||
|
||||
pool, err := pgxpool.NewWithConfig(ctx, poolCfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("создание пула: %w", err)
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"high-loud-apps-labs/internal/telemetry"
|
||||
)
|
||||
|
||||
type poolTracer struct {
|
||||
tracer trace.Tracer
|
||||
duration metric.Float64Histogram
|
||||
waiting metric.Int64UpDownCounter
|
||||
}
|
||||
|
||||
func newPoolTracer() (*poolTracer, error) {
|
||||
meter := telemetry.Meter("storage.postgres")
|
||||
duration, err := meter.Float64Histogram("db_pool_acquire_duration_seconds",
|
||||
metric.WithUnit("s"),
|
||||
metric.WithExplicitBucketBoundaries(0.0001, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5),
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
waiting, err := meter.Int64UpDownCounter("db_pool_waiting",
|
||||
metric.WithDescription("Число незавершённых вызовов Acquire"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &poolTracer{tracer: telemetry.Tracer("storage.postgres"), duration: duration, waiting: waiting}, nil
|
||||
}
|
||||
|
||||
type acquireStartKey struct{}
|
||||
|
||||
func (t *poolTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
|
||||
ctx, _ = t.tracer.Start(ctx, "postgresql.acquire")
|
||||
t.waiting.Add(ctx, 1)
|
||||
return context.WithValue(ctx, acquireStartKey{}, time.Now())
|
||||
}
|
||||
|
||||
func (t *poolTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
|
||||
t.waiting.Add(ctx, -1)
|
||||
t.duration.Record(ctx, time.Since(ctx.Value(acquireStartKey{}).(time.Time)).Seconds())
|
||||
endSpan(ctx, data.Err)
|
||||
}
|
||||
|
||||
func (t *poolTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context {
|
||||
// Передаём только известный SQL-глагол, без текста запроса и параметров.
|
||||
op := "QUERY"
|
||||
sql := strings.TrimSpace(data.SQL)
|
||||
if i := strings.IndexAny(sql, " \r\n\t;("); i >= 0 {
|
||||
sql = sql[:i]
|
||||
}
|
||||
switch verb := strings.ToUpper(sql); verb {
|
||||
case "SELECT", "INSERT", "UPDATE", "DELETE", "BEGIN", "COMMIT", "ROLLBACK", "WITH":
|
||||
op = verb
|
||||
}
|
||||
ctx, _ = t.tracer.Start(ctx, "postgresql."+op, trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system.name", "postgresql"), attribute.String("db.operation.name", op)))
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (t *poolTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
||||
endSpan(ctx, data.Err)
|
||||
}
|
||||
|
||||
func endSpan(ctx context.Context, err error) {
|
||||
span := trace.SpanFromContext(ctx)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, "операция PostgreSQL не выполнена")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
|
||||
func ObservePool(pool *pgxpool.Pool) (metric.Registration, error) {
|
||||
meter := telemetry.Meter("storage.postgres")
|
||||
connections, err := meter.Int64ObservableGauge("db_pool_connections")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
|
||||
stat := pool.Stat()
|
||||
observe := func(state string, count int32) {
|
||||
observer.ObserveInt64(connections, int64(count), metric.WithAttributes(attribute.String("state", state)))
|
||||
}
|
||||
observe("used", stat.AcquiredConns())
|
||||
observe("idle", stat.IdleConns())
|
||||
observe("max", stat.MaxConns())
|
||||
return nil
|
||||
}, connections)
|
||||
}
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"high-loud-apps-labs/internal/telemetry"
|
||||
)
|
||||
|
||||
func NewClient(url string) (*redis.Client, error) {
|
||||
@@ -22,6 +24,7 @@ func NewClient(url string) (*redis.Client, error) {
|
||||
opt.ConnMaxIdleTime = time.Minute
|
||||
|
||||
client := redis.NewClient(opt)
|
||||
client.AddHook(tracingHook{tracer: telemetry.Tracer("storage.redis")})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
package redis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type tracingHook struct{ tracer trace.Tracer }
|
||||
|
||||
func (h tracingHook) DialHook(next redis.DialHook) redis.DialHook { return next }
|
||||
|
||||
func (h tracingHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
return h.run(ctx, cmd.Name(), func(ctx context.Context) error { return next(ctx, cmd) })
|
||||
}
|
||||
}
|
||||
|
||||
func (h tracingHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return func(ctx context.Context, cmds []redis.Cmder) error {
|
||||
return h.run(ctx, "pipeline", func(ctx context.Context) error { return next(ctx, cmds) })
|
||||
}
|
||||
}
|
||||
|
||||
func (h tracingHook) run(ctx context.Context, op string, call func(context.Context) error) error {
|
||||
ctx, span := h.tracer.Start(ctx, "redis."+op, trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("db.system.name", "redis"), attribute.String("db.operation.name", op)))
|
||||
defer span.End()
|
||||
err := call(ctx)
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
span.SetStatus(codes.Error, "операция Redis не выполнена")
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -3,9 +3,11 @@ package s3
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
|
||||
|
||||
"high-loud-apps-labs/internal/domain"
|
||||
)
|
||||
@@ -25,10 +27,25 @@ type MinioStorage struct {
|
||||
}
|
||||
|
||||
func New(ctx context.Context, cfg Config) (*MinioStorage, error) {
|
||||
base, err := minio.DefaultTransport(cfg.UseSSL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Оборачиваем базовый транспорт в otelhttp.
|
||||
// Опция WithSpanNameFormatter задаёт красивое имя спана (s3.PUT, s3.GET),
|
||||
// чтобы в Grafana не смотреть на дефолтное "HTTP PUT".
|
||||
tracedTransport := otelhttp.NewTransport(base,
|
||||
otelhttp.WithSpanNameFormatter(func(operation string, r *http.Request) string {
|
||||
return "s3." + r.Method
|
||||
}),
|
||||
)
|
||||
|
||||
client, err := minio.New(cfg.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
Transport: tracedTransport,
|
||||
Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""),
|
||||
Secure: cfg.UseSSL,
|
||||
Region: cfg.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
// Один span на HTTP-попытку MinIO до получения заголовков ответа.
|
||||
type tracingTransport struct {
|
||||
next http.RoundTripper
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
func (t tracingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
ctx, span := t.tracer.Start(req.Context(), "s3."+req.Method, trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("http.request.method", req.Method)))
|
||||
defer span.End()
|
||||
|
||||
resp, err := t.next.RoundTrip(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, "ошибка соединения с S3")
|
||||
} else {
|
||||
span.SetAttributes(attribute.Int("http.response.status_code", resp.StatusCode))
|
||||
if resp.StatusCode >= 400 {
|
||||
span.SetStatus(codes.Error, http.StatusText(resp.StatusCode))
|
||||
}
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
Reference in New Issue
Block a user