111 lines
4.3 KiB
Go
111 lines
4.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/metric"
|
|
)
|
|
|
|
type databaseTracer struct {
|
|
acquiring atomic.Int64
|
|
}
|
|
|
|
func (d *databaseTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
|
|
d.acquiring.Add(1)
|
|
return ctx
|
|
}
|
|
|
|
func (d *databaseTracer) TraceAcquireEnd(context.Context, *pgxpool.Pool, pgxpool.TraceAcquireEndData) {
|
|
d.acquiring.Add(-1)
|
|
}
|
|
|
|
func observeRuntime(m metric.Meter, role string, db *pgxpool.Pool, dbTrace *databaseTracer) error {
|
|
used, errUsed := m.Int64ObservableGauge("messenger_db_pool_used")
|
|
capacity, errMax := m.Int64ObservableGauge("messenger_db_pool_capacity")
|
|
waiting, errWaiting := m.Int64ObservableGauge("messenger_db_pool_acquiring")
|
|
waits, errWaits := m.Int64ObservableCounter("messenger_db_pool_waits")
|
|
cpu, errCPU := m.Float64ObservableCounter("messenger_container_cpu", metric.WithUnit("s"))
|
|
memory, errMemory := m.Int64ObservableGauge("messenger_container_memory", metric.WithUnit("By"))
|
|
depth, errDepth := m.Int64ObservableGauge("messenger_queue_depth")
|
|
age, errAge := m.Float64ObservableGauge("messenger_queue_oldest_age", metric.WithUnit("s"))
|
|
slo, errSLO := m.Int64ObservableCounter("messenger_attachment_slo_jobs")
|
|
if err := errors.Join(errUsed, errMax, errWaiting, errWaits, errCPU, errMemory, errDepth, errAge, errSLO); err != nil {
|
|
return err
|
|
}
|
|
instruments := []metric.Observable{used, capacity, waiting, waits, cpu, memory}
|
|
if role == "dispatcher" {
|
|
instruments = append(instruments, depth, age, slo)
|
|
}
|
|
_, err := m.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
|
|
stat := db.Stat()
|
|
o.ObserveInt64(used, int64(stat.AcquiredConns()))
|
|
o.ObserveInt64(capacity, int64(stat.MaxConns()))
|
|
o.ObserveInt64(waiting, dbTrace.acquiring.Load())
|
|
o.ObserveInt64(waits, stat.EmptyAcquireCount())
|
|
if data, err := os.ReadFile("/sys/fs/cgroup/cpu.stat"); err == nil {
|
|
for _, line := range strings.Split(string(data), "\n") {
|
|
if micros, found := strings.CutPrefix(line, "usage_usec "); found {
|
|
if value, err := strconv.ParseFloat(micros, 64); err == nil {
|
|
o.ObserveFloat64(cpu, value/1e6)
|
|
}
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if data, err := os.ReadFile("/sys/fs/cgroup/memory.current"); err == nil {
|
|
if value, err := strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64); err == nil {
|
|
o.ObserveInt64(memory, value)
|
|
}
|
|
}
|
|
if role != "dispatcher" {
|
|
return nil
|
|
}
|
|
sample, cancel := context.WithTimeout(ctx, 700*time.Millisecond)
|
|
defer cancel()
|
|
var pending, good, matured int64
|
|
var oldest float64
|
|
// Measure SQL backlog including jobs waiting for a retry or for XADD.
|
|
// Count each first publication exactly once, at its SLO deadline,
|
|
// even if the worker is down or its final state is failed.
|
|
err := db.QueryRow(sample, `
|
|
WITH sample AS (SELECT statement_timestamp() AS now)
|
|
SELECT count(*) FILTER (WHERE state='queued'),
|
|
coalesce(extract(epoch FROM (sample.now - min(created_at) FILTER (WHERE state='queued'))), 0)::float8,
|
|
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'),
|
|
count(*) FILTER (WHERE version=1 AND created_at <= sample.now - interval '5 seconds'
|
|
AND ready_at <= created_at + interval '5 seconds')
|
|
FROM sample LEFT JOIN image_jobs ON true GROUP BY sample.now`).Scan(&pending, &oldest, &matured, &good)
|
|
if err == nil {
|
|
queue := metric.WithAttributes(attribute.String("queue", imageStream))
|
|
o.ObserveInt64(depth, pending, queue)
|
|
o.ObserveFloat64(age, oldest, queue)
|
|
o.ObserveInt64(slo, good, metric.WithAttributes(attribute.String("outcome", "good")))
|
|
o.ObserveInt64(slo, matured-good, metric.WithAttributes(attribute.String("outcome", "missed")))
|
|
}
|
|
return nil
|
|
}, instruments...)
|
|
return err
|
|
}
|
|
|
|
func (t *telemetry) observeWorker(m metric.Meter) error {
|
|
used, errUsed := m.Int64ObservableGauge("messenger_worker_busy")
|
|
capacity, errCapacity := m.Int64ObservableGauge("messenger_worker_capacity")
|
|
if err := errors.Join(errUsed, errCapacity); err != nil {
|
|
return err
|
|
}
|
|
_, err := m.RegisterCallback(func(_ context.Context, o metric.Observer) error {
|
|
o.ObserveInt64(used, t.workerBusy.Load())
|
|
o.ObserveInt64(capacity, 1)
|
|
return nil
|
|
}, used, capacity)
|
|
return err
|
|
}
|