feat(reconciler): lab6 record stale leases, recovered jobs and leadership

This commit is contained in:
Голиков Maximillian Антон
2026-11-17 10:54:15 +03:00
parent ddd72a2181
commit 3460856dc9
2 changed files with 64 additions and 1 deletions
+16 -1
View File
@@ -23,6 +23,7 @@ type Reconciler struct {
pool *pgxpool.Pool
gracePeriod time.Duration
batchSize int
metrics *reconcilerMetrics
}
func NewReconciler(
@@ -42,6 +43,7 @@ func NewReconciler(
pool: pool,
gracePeriod: gracePeriod,
batchSize: 100,
metrics: newReconcilerMetrics(),
}
}
@@ -49,6 +51,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
if interval <= 0 {
interval = 5 * time.Minute
}
registration, err := r.metrics.observeLeader()
if err != nil {
log.Printf("reconciler: ошибка регистрации метрики лидерства: %v", err)
} else {
defer registration.Unregister()
}
ticker := time.NewTicker(interval)
defer ticker.Stop()
@@ -79,7 +88,11 @@ func (r *Reconciler) tick(ctx context.Context) {
if !acquired {
return
}
defer release()
r.metrics.leader.Store(true)
defer func() {
release()
r.metrics.leader.Store(false)
}()
leasesReset := r.sweepExpiredLeases(ctx)
stagingDeleted := r.sweepStaging(ctx)
@@ -129,6 +142,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
return 0
}
r.metrics.stale.Add(ctx, int64(len(expired)))
recovered := 0
for _, a := range expired {
if err := r.attachments.ResetLease(ctx, a.ID); err != nil {
@@ -156,6 +170,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
}
recovered++
}
r.metrics.recovered.Add(ctx, int64(recovered))
return recovered
}
@@ -0,0 +1,48 @@
package service
import (
"context"
"sync/atomic"
"go.opentelemetry.io/otel/metric"
"high-loud-apps-labs/internal/telemetry"
)
type reconcilerMetrics struct {
stale metric.Int64Counter
recovered metric.Int64Counter
leader atomic.Bool
}
func newReconcilerMetrics() *reconcilerMetrics {
meter := telemetry.Meter("reconciler")
stale, err := meter.Int64Counter("reconciler_stale_leases_total",
metric.WithDescription("Число наблюдений просроченных аренд"))
if err != nil {
panic(err)
}
recovered, err := meter.Int64Counter("reconciler_recovered_jobs_total",
metric.WithDescription("Число задач после успешного сброса аренды и записи в outbox"))
if err != nil {
panic(err)
}
return &reconcilerMetrics{stale: stale, recovered: recovered}
}
func (m *reconcilerMetrics) observeLeader() (metric.Registration, error) {
meter := telemetry.Meter("reconciler")
gauge, err := meter.Int64ObservableGauge("reconciler_leader",
metric.WithDescription("1, пока этот экземпляр держит advisory lock"))
if err != nil {
return nil, err
}
return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
var value int64
if m.leader.Load() {
value = 1
}
o.ObserveInt64(gauge, value)
return nil
}, gauge)
}