diff --git a/backend/cmd/worker/main.go b/backend/cmd/worker/main.go index 1986dc0..2102d5d 100644 --- a/backend/cmd/worker/main.go +++ b/backend/cmd/worker/main.go @@ -13,6 +13,7 @@ import ( amqp "github.com/rabbitmq/amqp091-go" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "high-loud-apps-labs/internal/config" @@ -80,11 +81,26 @@ func main() { os.Exit(1) } + meter := telemetry.Meter("worker.consumer") + busy, err := meter.Int64UpDownCounter("worker_busy_slots", metric.WithDescription("Занятые слоты worker")) + if err != nil { + slog.Error("Ошибка регистрации метрики занятости worker", "error", err) + os.Exit(1) + } + capacity, err := meter.Int64Gauge("worker_capacity_slots", metric.WithDescription("Число слотов worker")) + if err != nil { + slog.Error("Ошибка регистрации метрики ёмкости worker", "error", err) + os.Exit(1) + } + // Цикл обрабатывает одну доставку за раз. Prefetch не задаёт параллельность. + capacity.Record(ctx, 1) + busy.Add(ctx, 0) + pipeline := worker.NewPipeline(store.Attachments(), objectStorage, store.Variants(), store.FailedJobs(), store.Outbox()) // Цикл переподключения к RabbitMQ for { - err := runWorker(ctx, cfg, pipeline) + err := runWorker(ctx, cfg, pipeline, busy) if err == nil { break // Нормальное завершение по сигналу } @@ -93,7 +109,7 @@ func main() { } } -func runWorker(ctx context.Context, cfg *config.Config, pipeline *worker.Pipeline) error { +func runWorker(ctx context.Context, cfg *config.Config, pipeline *worker.Pipeline, busy metric.Int64UpDownCounter) error { conn, err := amqp.Dial(cfg.RabbitMQURL) if err != nil { return err @@ -133,7 +149,10 @@ func runWorker(ctx context.Context, cfg *config.Config, pipeline *worker.Pipelin if !ok { return context.Canceled // Канал закрыт (соединение разорвано) } - if err := consume(ctx, d, pipeline); err != nil { + busy.Add(ctx, 1) + err := consume(ctx, d, pipeline) + busy.Add(ctx, -1) + if err != nil { return err } } diff --git a/backend/internal/telemetry/process_linux.go b/backend/internal/telemetry/process_linux.go new file mode 100644 index 0000000..ba37600 --- /dev/null +++ b/backend/internal/telemetry/process_linux.go @@ -0,0 +1,45 @@ +//go:build linux + +package telemetry + +import ( + "context" + "os" + "strconv" + "strings" + "syscall" + + "go.opentelemetry.io/otel/metric" +) + +func registerProcessMetrics(meter metric.Meter) error { + cpu, err := meter.Float64ObservableCounter("process_cpu_seconds_total", metric.WithUnit("s")) + if err != nil { + return err + } + rss, err := meter.Int64ObservableGauge("process_resident_memory_bytes", metric.WithUnit("By")) + if err != nil { + return err + } + _, err = meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error { + var usage syscall.Rusage + if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil { + seconds := float64(usage.Utime.Sec+usage.Stime.Sec) + + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6 + observer.ObserveFloat64(cpu, seconds) + } + + data, err := os.ReadFile("/proc/self/statm") + if err != nil { + return nil + } + fields := strings.Fields(string(data)) + if len(fields) >= 2 { + if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil { + observer.ObserveInt64(rss, pages*int64(os.Getpagesize())) + } + } + return nil + }, cpu, rss) + return err +} diff --git a/backend/internal/telemetry/process_other.go b/backend/internal/telemetry/process_other.go new file mode 100644 index 0000000..eb19a7d --- /dev/null +++ b/backend/internal/telemetry/process_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package telemetry + +import "go.opentelemetry.io/otel/metric" + +func registerProcessMetrics(metric.Meter) error { return nil } diff --git a/backend/internal/telemetry/telemetry.go b/backend/internal/telemetry/telemetry.go index 6970190..44da3d1 100644 --- a/backend/internal/telemetry/telemetry.go +++ b/backend/internal/telemetry/telemetry.go @@ -106,6 +106,9 @@ func Setup(ctx context.Context, cfg SetupConfig) (*Providers, error) { otel.SetTracerProvider(tp) otel.SetMeterProvider(mp) + if err := registerProcessMetrics(mp.Meter("process")); err != nil { + slog.Warn("Не удалось зарегистрировать метрики процесса", "error", err) + } // Пропагация контекста через W3C Trace Context и Baggage otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(