From 3460856dc9cc8b685c355e266c574d87f38be440 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=93=D0=BE=D0=BB=D0=B8=D0=BA=D0=BE=D0=B2=20Maximillian?= =?UTF-8?q?=20=D0=90=D0=BD=D1=82=D0=BE=D0=BD?= Date: Tue, 17 Nov 2026 10:54:15 +0300 Subject: [PATCH] feat(reconciler): lab6 record stale leases, recovered jobs and leadership --- backend/internal/service/reconciler.go | 17 ++++++- .../internal/service/reconciler_metrics.go | 48 +++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 backend/internal/service/reconciler_metrics.go diff --git a/backend/internal/service/reconciler.go b/backend/internal/service/reconciler.go index 5b8552b..1034c62 100644 --- a/backend/internal/service/reconciler.go +++ b/backend/internal/service/reconciler.go @@ -23,6 +23,7 @@ type Reconciler struct { pool *pgxpool.Pool gracePeriod time.Duration batchSize int + metrics *reconcilerMetrics } func NewReconciler( @@ -42,6 +43,7 @@ func NewReconciler( pool: pool, gracePeriod: gracePeriod, batchSize: 100, + metrics: newReconcilerMetrics(), } } @@ -49,6 +51,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) { if interval <= 0 { interval = 5 * time.Minute } + registration, err := r.metrics.observeLeader() + if err != nil { + log.Printf("reconciler: ошибка регистрации метрики лидерства: %v", err) + } else { + defer registration.Unregister() + } + ticker := time.NewTicker(interval) defer ticker.Stop() @@ -79,7 +88,11 @@ func (r *Reconciler) tick(ctx context.Context) { if !acquired { return } - defer release() + r.metrics.leader.Store(true) + defer func() { + release() + r.metrics.leader.Store(false) + }() leasesReset := r.sweepExpiredLeases(ctx) stagingDeleted := r.sweepStaging(ctx) @@ -129,6 +142,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int { return 0 } + r.metrics.stale.Add(ctx, int64(len(expired))) recovered := 0 for _, a := range expired { if err := r.attachments.ResetLease(ctx, a.ID); err != nil { @@ -156,6 +170,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int { } recovered++ } + r.metrics.recovered.Add(ctx, int64(recovered)) return recovered } diff --git a/backend/internal/service/reconciler_metrics.go b/backend/internal/service/reconciler_metrics.go new file mode 100644 index 0000000..799648d --- /dev/null +++ b/backend/internal/service/reconciler_metrics.go @@ -0,0 +1,48 @@ +package service + +import ( + "context" + "sync/atomic" + + "go.opentelemetry.io/otel/metric" + + "high-loud-apps-labs/internal/telemetry" +) + +type reconcilerMetrics struct { + stale metric.Int64Counter + recovered metric.Int64Counter + leader atomic.Bool +} + +func newReconcilerMetrics() *reconcilerMetrics { + meter := telemetry.Meter("reconciler") + stale, err := meter.Int64Counter("reconciler_stale_leases_total", + metric.WithDescription("Число наблюдений просроченных аренд")) + if err != nil { + panic(err) + } + recovered, err := meter.Int64Counter("reconciler_recovered_jobs_total", + metric.WithDescription("Число задач после успешного сброса аренды и записи в outbox")) + if err != nil { + panic(err) + } + return &reconcilerMetrics{stale: stale, recovered: recovered} +} + +func (m *reconcilerMetrics) observeLeader() (metric.Registration, error) { + meter := telemetry.Meter("reconciler") + gauge, err := meter.Int64ObservableGauge("reconciler_leader", + metric.WithDescription("1, пока этот экземпляр держит advisory lock")) + if err != nil { + return nil, err + } + return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error { + var value int64 + if m.leader.Load() { + value = 1 + } + o.ObserveInt64(gauge, value) + return nil + }, gauge) +}