Files
mtusi2026lab6-fork/telemetry_runtime.go
T
2026-11-19 19:24:45 +03:00

111 lines
4.3 KiB
Go

package main
import (
"context"
"errors"
"os"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
type databaseTracer struct {
acquiring atomic.Int64
}
func (d *databaseTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
d.acquiring.Add(1)
return ctx
}
func (d *databaseTracer) TraceAcquireEnd(context.Context, *pgxpool.Pool, pgxpool.TraceAcquireEndData) {
d.acquiring.Add(-1)
}
func observeRuntime(m metric.Meter, role string, db *pgxpool.Pool, dbTrace *databaseTracer) error {
used, errUsed := m.Int64ObservableGauge("messenger_db_pool_used")
capacity, errMax := m.Int64ObservableGauge("messenger_db_pool_capacity")
waiting, errWaiting := m.Int64ObservableGauge("messenger_db_pool_acquiring")
waits, errWaits := m.Int64ObservableCounter("messenger_db_pool_waits")
cpu, errCPU := m.Float64ObservableCounter("messenger_container_cpu", metric.WithUnit("s"))
memory, errMemory := m.Int64ObservableGauge("messenger_container_memory", metric.WithUnit("By"))
depth, errDepth := m.Int64ObservableGauge("messenger_queue_depth")
age, errAge := m.Float64ObservableGauge("messenger_queue_oldest_age", metric.WithUnit("s"))
slo, errSLO := m.Int64ObservableCounter("messenger_attachment_slo_jobs")
if err := errors.Join(errUsed, errMax, errWaiting, errWaits, errCPU, errMemory, errDepth, errAge, errSLO); err != nil {
return err
}
instruments := []metric.Observable{used, capacity, waiting, waits, cpu, memory}
if role == "dispatcher" {
instruments = append(instruments, depth, age, slo)
}
_, err := m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
stat := db.Stat()
o.ObserveInt64(used, int64(stat.AcquiredConns()))
o.ObserveInt64(capacity, int64(stat.MaxConns()))
o.ObserveInt64(waiting, dbTrace.acquiring.Load())
o.ObserveInt64(waits, stat.EmptyAcquireCount())
if data, err := os.ReadFile("/sys/fs/cgroup/cpu.stat"); err == nil {
for _, line := range strings.Split(string(data), "\n") {
if micros, found := strings.CutPrefix(line, "usage_usec "); found {
if value, err := strconv.ParseFloat(micros, 64); err == nil {
o.ObserveFloat64(cpu, value/1e6)
}
break
}
}
}
if data, err := os.ReadFile("/sys/fs/cgroup/memory.current"); err == nil {
if value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil {
o.ObserveInt64(memory, value)
}
}
if role != "dispatcher" {
return nil
}
sample, cancel := context.WithTimeout(ctx, 700*time.Millisecond)
defer cancel()
var pending, good, matured int64
var oldest float64
// Measure SQL backlog including jobs waiting for a retry or for XADD.
// Count each first publication exactly once, at its SLO deadline,
// even if the worker is down or its final state is failed.
err := db.QueryRow(sample, `
WITH sample AS (SELECT statement_timestamp() AS now)
SELECT count(*) FILTER (WHERE state='queued'),
coalesce(extract(epoch FROM (sample.now - min(created_at) FILTER (WHERE state='queued'))), 0)::float8,
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'),
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'
AND ready_at <= created_at + interval '5 seconds')
FROM sample LEFT JOIN image_jobs ON true GROUP BY sample.now`).Scan(&pending, &oldest, &matured, &good)
if err == nil {
queue := metric.WithAttributes(attribute.String("queue", imageStream))
o.ObserveInt64(depth, pending, queue)
o.ObserveFloat64(age, oldest, queue)
o.ObserveInt64(slo, good, metric.WithAttributes(attribute.String("outcome", "good")))
o.ObserveInt64(slo, matured-good, metric.WithAttributes(attribute.String("outcome", "missed")))
}
return nil
}, instruments...)
return err
}
func (t *telemetry) observeWorker(m metric.Meter) error {
used, errUsed := m.Int64ObservableGauge("messenger_worker_busy")
capacity, errCapacity := m.Int64ObservableGauge("messenger_worker_capacity")
if err := errors.Join(errUsed, errCapacity); err != nil {
return err
}
_, err := m.RegisterCallback(func(_ context.Context, o metric.Observer) error {
o.ObserveInt64(used, t.workerBusy.Load())
o.ObserveInt64(capacity, 1)
return nil
}, used, capacity)
return err
}