forked from mtusi2026/mtusi2026lab6
feat(reconciler): lab6 record stale leases, recovered jobs and leadership
This commit is contained in:
@@ -23,6 +23,7 @@ type Reconciler struct {
|
||||
pool *pgxpool.Pool
|
||||
gracePeriod time.Duration
|
||||
batchSize int
|
||||
metrics *reconcilerMetrics
|
||||
}
|
||||
|
||||
func NewReconciler(
|
||||
@@ -42,6 +43,7 @@ func NewReconciler(
|
||||
pool: pool,
|
||||
gracePeriod: gracePeriod,
|
||||
batchSize: 100,
|
||||
metrics: newReconcilerMetrics(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +51,13 @@ func (r *Reconciler) Run(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = 5 * time.Minute
|
||||
}
|
||||
registration, err := r.metrics.observeLeader()
|
||||
if err != nil {
|
||||
log.Printf("reconciler: ошибка регистрации метрики лидерства: %v", err)
|
||||
} else {
|
||||
defer registration.Unregister()
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
|
||||
@@ -79,7 +88,11 @@ func (r *Reconciler) tick(ctx context.Context) {
|
||||
if !acquired {
|
||||
return
|
||||
}
|
||||
defer release()
|
||||
r.metrics.leader.Store(true)
|
||||
defer func() {
|
||||
release()
|
||||
r.metrics.leader.Store(false)
|
||||
}()
|
||||
|
||||
leasesReset := r.sweepExpiredLeases(ctx)
|
||||
stagingDeleted := r.sweepStaging(ctx)
|
||||
@@ -129,6 +142,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
r.metrics.stale.Add(ctx, int64(len(expired)))
|
||||
recovered := 0
|
||||
for _, a := range expired {
|
||||
if err := r.attachments.ResetLease(ctx, a.ID); err != nil {
|
||||
@@ -156,6 +170,7 @@ func (r *Reconciler) sweepExpiredLeases(ctx context.Context) int {
|
||||
}
|
||||
recovered++
|
||||
}
|
||||
r.metrics.recovered.Add(ctx, int64(recovered))
|
||||
return recovered
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync/atomic"
|
||||
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
|
||||
"high-loud-apps-labs/internal/telemetry"
|
||||
)
|
||||
|
||||
type reconcilerMetrics struct {
|
||||
stale metric.Int64Counter
|
||||
recovered metric.Int64Counter
|
||||
leader atomic.Bool
|
||||
}
|
||||
|
||||
func newReconcilerMetrics() *reconcilerMetrics {
|
||||
meter := telemetry.Meter("reconciler")
|
||||
stale, err := meter.Int64Counter("reconciler_stale_leases_total",
|
||||
metric.WithDescription("Число наблюдений просроченных аренд"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
recovered, err := meter.Int64Counter("reconciler_recovered_jobs_total",
|
||||
metric.WithDescription("Число задач после успешного сброса аренды и записи в outbox"))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &reconcilerMetrics{stale: stale, recovered: recovered}
|
||||
}
|
||||
|
||||
func (m *reconcilerMetrics) observeLeader() (metric.Registration, error) {
|
||||
meter := telemetry.Meter("reconciler")
|
||||
gauge, err := meter.Int64ObservableGauge("reconciler_leader",
|
||||
metric.WithDescription("1, пока этот экземпляр держит advisory lock"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return meter.RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
||||
var value int64
|
||||
if m.leader.Load() {
|
||||
value = 1
|
||||
}
|
||||
o.ObserveInt64(gauge, value)
|
||||
return nil
|
||||
}, gauge)
|
||||
}
|
||||
Reference in New Issue
Block a user