feat: add pipeline tracing and observability alerts

This commit is contained in:
Dmitrii Grishko
2026-11-13 18:52:25 +03:00
parent 549edc8d7a
commit d753e348eb
19 changed files with 821 additions and 21 deletions
+2 -1
View File
@@ -203,7 +203,7 @@ services:
restart: unless-stopped
cpus: 0.25
mem_limit: 128m
networks: [data]
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
command: ["./messenger", "dispatcher"]
@@ -334,6 +334,7 @@ services:
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
- prometheus_data:/prometheus
loki:
+20
View File
@@ -0,0 +1,20 @@
groups:
- name: messenger
interval: 5s
rules:
- alert: MessengerHighErrorRate
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
for: 15s
labels:
severity: warning
annotations:
summary: More than 5% of business requests return 5xx over one minute
runbook: Check API error spans and PostgreSQL operations in Tempo.
- alert: MessengerQueueLag
expr: max(messenger_queue_oldest_seconds) > 20
for: 15s
labels:
severity: warning
annotations:
summary: Oldest outstanding attachment job is over 20 seconds old
runbook: Check outstanding jobs, worker saturation and consumer traces.
@@ -200,6 +200,314 @@
},
"overrides": []
}
},
{
"id": 7,
"title": "API · p50 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 8,
"title": "API · p99 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 24,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 9,
"title": "API · доля 5xx",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 10,
"title": "Очередь attachments.v1 · незавершённые задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 32,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_depth)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 11,
"title": "Очередь attachments.v1 · возраст старейшего задания",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "max(messenger_queue_oldest_seconds)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 12,
"title": "PostgreSQL · занятые соединения",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 40,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_db_pool_used)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 13,
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "ops"
},
"overrides": []
}
},
{
"id": 14,
"title": "Worker · загрузка",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 48,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "percentunit"
},
"overrides": []
}
},
{
"id": 15,
"title": "Вложения · p95 до ready",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 16,
"title": "Процессы · загрузка CPU",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 12,
"y": 56,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "cores"
},
"overrides": []
}
},
{
"id": 17,
"title": "Процессы · резидентная память",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"gridPos": {
"x": 0,
"y": 64,
"w": 12,
"h": 8
},
"targets": [
{
"refId": "A",
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
"legendFormat": "{{service_name}}"
}
],
"fieldConfig": {
"defaults": {
"unit": "bytes"
},
"overrides": []
}
}
]
}
+2
View File
@@ -1,5 +1,7 @@
global:
scrape_interval: 5s
rule_files:
- /etc/prometheus/alerts.yaml
scrape_configs:
- job_name: otel-collector
static_configs:
+27
View File
@@ -0,0 +1,27 @@
# Диагностика API и обработки вложений
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
```promql
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
```
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
Команды локальной проверки после `make app-up`:
```sh
python3 scripts/check_observability.py pipeline
python3 scripts/check_observability_incidents.py worker
python3 scripts/check_observability_incidents.py database
python3 scripts/check_observability_incidents.py retry
python3 scripts/check_observability_incidents.py telemetry
```
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
+31 -2
View File
@@ -31,7 +31,7 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
args = parser.parse_args()
def query(path, **params):
@@ -47,13 +47,42 @@ def main():
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
if args.criterion == "pipeline":
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
client.wait_ready(bob, upload["json"])
request_id = upload["headers"]["X-Request-Id"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
start=start, end=time.time_ns())["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
trace = source("tempo", "/api/traces/" + trace_id)
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
if producer and consumer:
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
"worker S3 and PostgreSQL spans missing")
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
return
time.sleep(2)
raise Failure("upload trace did not include the worker pipeline")
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
import argparse
import json
from pathlib import Path
import re
import subprocess
import sys
import time
import urllib.parse
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
client = Client(args.base_url)
alice, _, _, chat = client.bootstrap()
def trace_for(request_id, matches):
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
deadline = time.monotonic() + 35
while time.monotonic() < deadline:
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
logs = json.load(response)["data"]["result"]
if logs:
trace_id = logs[0]["stream"]["trace_id"]
try:
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
trace = json.load(response)
except urllib.error.HTTPError as error:
if error.code != 404:
raise
else:
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
if matches(spans):
return trace_id
time.sleep(2)
raise Failure("expected trace was not found")
def alerts(name):
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
with urllib.request.urlopen(endpoint, timeout=5) as response:
active = json.load(response)["data"]["alerts"]
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
def wait_alert(name, firing, exercise=lambda: None):
deadline = time.monotonic() + 100
while time.monotonic() < deadline:
exercise()
if alerts(name) == firing:
print(name, "firing" if firing else "resolved", flush=True)
return
time.sleep(2)
raise Failure(name + ": expected alert state not observed")
if args.incident == "worker":
compose("stop", "worker")
try:
attachment, _ = client.attachment(alice, chat)
wait_alert("MessengerQueueLag", True)
finally:
compose("start", "worker")
client.wait_ready(alice, attachment)
wait_alert("MessengerQueueLag", False)
elif args.incident == "database":
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
marker = "lab6-fault-" + uuid.uuid4().hex
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
"sh", marker]
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
try:
time.sleep(2)
request_ids = []
def exercise():
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
request_ids.append(response["headers"]["X-Request-Id"])
wait_alert("MessengerHighErrorRate", True, exercise)
finally:
# Terminate only the synthetic lock session so the transaction rolls back immediately.
compose("exec", "-T", "db", "sh", "-c",
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
lock.wait(timeout=10)
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
trace_id = trace_for(request_ids[-1], lambda spans: any(
span["name"] == "postgresql query"
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
wait_alert("MessengerHighErrorRate", False)
elif args.incident == "retry":
compose("stop", "worker")
try:
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
headers={"Content-Type": "image/png"}, schema="Attachment")
attachment = upload["json"]
compose("stop", "s3")
try:
compose("up", "-d", "--no-deps", "worker")
deadline = time.monotonic() + 25
while time.monotonic() < deadline:
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
output = subprocess.check_output(
["docker", "compose", "exec", "-T", "db", "sh", "-c",
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
cwd=ROOT, text=True, timeout=10)
if int(output.strip()) > 0:
break
time.sleep(1)
else:
raise Failure("durable retry was not observed")
finally:
compose("up", "-d", "--wait", "s3")
client.wait_ready(alice, attachment)
def retried(spans):
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
return len(set(attempts)) >= 2
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
finally:
compose("start", "worker")
else:
since = str(int(time.time()))
compose("stop", "tempo")
try:
for _ in range(8):
client.message(alice, chat, "telemetry-backend-off")
time.sleep(5)
attachment, _ = client.attachment(alice, chat)
client.wait_ready(alice, attachment)
compose("stats", "--no-stream", "otel-collector")
print("API и worker работали при остановленном Tempo.", flush=True)
finally:
compose("start", "tempo")
output = subprocess.check_output(
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
cwd=ROOT, text=True, timeout=15)
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
print("FAIL:", error, file=sys.stderr)
sys.exit(1)
+3
View File
@@ -94,6 +94,9 @@ func RunHTTP() error {
return fmt.Errorf("telemetry initialization failed: %w", err)
}
defer stopTelemetry()
if err := signals.ObservePool(db); err != nil {
return err
}
server := &http.Server{
Addr: serverAddress(),
+7 -1
View File
@@ -2,7 +2,9 @@ package repositories
import (
"context"
"errors"
"messenger/src/backend/telemetry"
"net"
"time"
@@ -53,8 +55,12 @@ func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
// mandatory catches an unroutable message; confirms alone do not prove routing.
headers := amqp.Table{}
for key, value := range telemetry.TraceHeaders(ctx) {
headers[key] = value
}
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body,
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
})
if err != nil {
return errors.New("RabbitMQ publication failed")
+3
View File
@@ -10,6 +10,8 @@ import (
"strings"
"time"
"messenger/src/backend/telemetry"
"github.com/golang-migrate/migrate/v4"
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
"github.com/golang-migrate/migrate/v4/source/iofs"
@@ -43,6 +45,7 @@ func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
if err != nil {
return nil, err
}
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
cfg.MaxConns = 10
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
maxConns, parseErr := strconv.Atoi(value)
+25 -6
View File
@@ -5,10 +5,14 @@ import (
"crypto/rand"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"math/big"
"time"
"messenger/src/backend/telemetry"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel/trace"
)
type Variant struct {
@@ -27,12 +31,14 @@ type Job struct {
PipelineVersion int `json:"pipeline_version"`
Attempt int `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"-"`
Tracestate string `json:"-"`
}
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
// Re-publication is intentional: a durable intent is never removed by publish.
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
if err != nil {
return nil, err
}
@@ -40,7 +46,7 @@ func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
jobs := []Job{}
for rows.Next() {
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt); err != nil {
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt, &j.Traceparent, &j.Tracestate); err != nil {
return nil, err
}
j.CreatedAt = j.CreatedAt.UTC()
@@ -104,7 +110,9 @@ var ErrProcessingTimeout = errors.New("processing timeout")
// ProcessJob holds the row lock across deterministic object writes. The lock and
// token check prevent overlapping/stale attempts from publishing metadata.
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) (result error) {
ctx, finish := telemetry.Operation(ctx, "postgresql finalize attachment", trace.SpanKindClient)
defer func() { finish(result) }()
tx, err := s.db.Begin(ctx)
if err != nil {
return err
@@ -202,7 +210,8 @@ func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) er
if err != nil {
return err
}
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, uuid.NewString(), a.ID, staging)
headers := telemetry.TraceHeaders(ctx)
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,traceparent,tracestate) VALUES($1,$2,$3,$4,$5)`, uuid.NewString(), a.ID, staging, headers["traceparent"], headers["tracestate"])
return err
}
@@ -211,3 +220,13 @@ func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, err
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
return done, err
}
// PendingJobs counts durable outstanding work, including leases and scheduled retries.
func (s *AttachmentStore) PendingJobs(ctx context.Context) (int64, float64, error) {
var count int64
var age float64
err := s.db.QueryRow(ctx, `SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now()-min(j.created_at)),0)::double precision
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing')`).Scan(&count, &age)
return count, age, err
}
@@ -0,0 +1 @@
ALTER TABLE attachment_jobs DROP COLUMN tracestate, DROP COLUMN traceparent;
@@ -0,0 +1,3 @@
ALTER TABLE attachment_jobs
ADD COLUMN traceparent text NOT NULL DEFAULT '',
ADD COLUMN tracestate text NOT NULL DEFAULT '';
+94
View File
@@ -0,0 +1,94 @@
package telemetry
import (
"context"
"go.opentelemetry.io/otel/attribute"
"os"
"strconv"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/metric"
)
func (s *Signals) ObservePool(pool *pgxpool.Pool) error {
if s == nil {
return nil
}
used, err := s.meter.Int64ObservableGauge("messenger.db.pool.used")
if err != nil {
return err
}
capacity, err := s.meter.Int64ObservableGauge("messenger.db.pool.capacity")
if err != nil {
return err
}
waited, err := s.meter.Int64ObservableCounter("messenger.db.pool.waited")
if err != nil {
return err
}
_, err = s.meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
stats := pool.Stat()
observer.ObserveInt64(used, int64(stats.AcquiredConns()))
observer.ObserveInt64(capacity, int64(stats.MaxConns()))
observer.ObserveInt64(waited, stats.EmptyAcquireCount())
return nil
}, used, capacity, waited)
return err
}
func (s *Signals) ObserveBacklog(read func(context.Context) (int64, float64, error)) error {
if s == nil {
return nil
}
depth, err := s.meter.Int64ObservableGauge("messenger.queue.depth")
if err != nil {
return err
}
age, err := s.meter.Float64ObservableGauge("messenger.queue.oldest", metric.WithUnit("s"))
if err != nil {
return err
}
_, err = s.meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error {
ctx, cancel := context.WithTimeout(ctx, time.Second)
defer cancel()
count, seconds, err := read(ctx)
if err == nil {
queue := metric.WithAttributes(attribute.String("queue.name", "attachments.v1"))
observer.ObserveInt64(depth, count, queue)
observer.ObserveFloat64(age, seconds, queue)
}
return nil
}, depth, age)
return err
}
// ObserveProcess exports Linux process CPU time and resident memory.
func (s *Signals) ObserveProcess() error {
cpu, err := s.meter.Float64ObservableCounter("messenger.process.cpu", metric.WithUnit("s"))
if err != nil {
return err
}
rss, err := s.meter.Int64ObservableGauge("messenger.process.rss", metric.WithUnit("By"))
if err != nil {
return err
}
_, err = s.meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
var usage syscall.Rusage
if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
observer.ObserveFloat64(cpu, float64(usage.Utime.Nano()+usage.Stime.Nano())/float64(time.Second))
}
if data, err := os.ReadFile("/proc/self/statm"); err == nil {
fields := strings.Fields(string(data))
if len(fields) >= 2 {
if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
observer.ObserveInt64(rss, pages*int64(os.Getpagesize()))
}
}
}
return nil
}, cpu, rss)
return err
}
+23
View File
@@ -0,0 +1,23 @@
package telemetry
import (
"context"
"github.com/jackc/pgx/v5"
"go.opentelemetry.io/otel/trace"
)
type PostgresTracer struct{}
type queryFinishKey struct{}
func (PostgresTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
ctx, finish := Operation(ctx, "postgresql query", trace.SpanKindClient)
return context.WithValue(ctx, queryFinishKey{}, finish)
}
func (PostgresTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
if finish, ok := ctx.Value(queryFinishKey{}).(func(error)); ok {
finish(data.Err)
}
}
+47
View File
@@ -0,0 +1,47 @@
package telemetry
import (
"context"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
)
func TraceHeaders(ctx context.Context) map[string]string {
headers := propagation.MapCarrier{}
propagation.TraceContext{}.Inject(ctx, headers)
return headers
}
func WithTraceHeaders(ctx context.Context, headers map[string]string) context.Context {
return propagation.TraceContext{}.Extract(ctx, propagation.MapCarrier(headers))
}
func (s *Signals) Publish(ctx context.Context, publish func(context.Context) error) error {
if s == nil {
return publish(ctx)
}
ctx, span := s.tracer.Start(ctx, "attachments.v1 publish", trace.WithSpanKind(trace.SpanKindProducer))
defer span.End()
err := publish(ctx)
if err != nil {
span.SetStatus(codes.Error, "")
}
return err
}
// Operation creates a child of the active span without recording arguments or raw errors.
func Operation(ctx context.Context, name string, kind trace.SpanKind) (context.Context, func(error)) {
parent := trace.SpanFromContext(ctx)
if !parent.IsRecording() {
return ctx, func(error) {}
}
ctx, span := parent.TracerProvider().Tracer("messenger").Start(ctx, name, trace.WithSpanKind(kind))
return ctx, func(err error) {
if err != nil {
span.SetStatus(codes.Error, "")
}
span.End()
}
}
+19 -2
View File
@@ -22,6 +22,9 @@ import (
)
type Signals struct {
meter metric.Meter
busy metric.Int64UpDownCounter
ready metric.Float64Histogram
tracer trace.Tracer
logger otellog.Logger
requests metric.Int64Counter
@@ -79,11 +82,25 @@ func Start(service string) (*Signals, func(), error) {
httpDuration, errHTTP := meter.Float64Histogram("messenger.http.duration", metric.WithUnit("s"), buckets)
jobs, errJobs := meter.Int64Counter("messenger.worker.jobs")
jobDuration, errJobDuration := meter.Float64Histogram("messenger.worker.duration", metric.WithUnit("s"), buckets)
if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration); err != nil {
busy, errBusy := meter.Int64UpDownCounter("messenger.worker.busy")
capacity, errCapacity := meter.Int64UpDownCounter("messenger.worker.capacity")
ready, errReady := meter.Float64Histogram("messenger.attachment.ready", metric.WithUnit("s"), buckets)
if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration, errBusy, errCapacity, errReady); err != nil {
stop()
return nil, func() {}, err
}
return &Signals{tp.Tracer("messenger"), lp.Logger("messenger"), requests, httpDuration, jobs, jobDuration}, stop, nil
if service == "messenger-worker" {
capacity.Add(ctx, 1)
busy.Add(ctx, 0)
}
signals := &Signals{meter: meter, busy: busy, ready: ready, tracer: tp.Tracer("messenger"),
logger: lp.Logger("messenger"), requests: requests, httpDuration: httpDuration,
jobs: jobs, jobDuration: jobDuration}
if err := signals.ObserveProcess(); err != nil {
stop()
return nil, func() {}, err
}
return signals, stop, nil
}
func (s *Signals) event(ctx context.Context, event string, failed bool, attrs ...otellog.KeyValue) {
+7 -2
View File
@@ -11,14 +11,19 @@ import (
"go.opentelemetry.io/otel/trace"
)
func (s *Signals) Job(ctx context.Context, jobType, attachmentID string) (context.Context, func(string)) {
func (s *Signals) Job(ctx context.Context, jobType, attachmentID string, attempt int, accepted time.Time) (context.Context, func(string)) {
if s == nil {
return ctx, func(string) {}
}
start := time.Now()
ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer))
s.busy.Add(ctx, 1)
ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(attribute.Int("job.attempt", attempt)))
return ctx, func(outcome string) {
defer span.End()
s.busy.Add(ctx, -1)
if outcome == "success" {
s.ready.Record(ctx, time.Since(accepted).Seconds())
}
attrs := []attribute.KeyValue{attribute.String("job.type", jobType), attribute.String("job.outcome", outcome)}
span.SetAttributes(attrs...)
failed := outcome == "error"
+28 -7
View File
@@ -11,6 +11,8 @@ import (
"syscall"
"time"
"go.opentelemetry.io/otel/trace"
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
@@ -60,19 +62,28 @@ func RunPipeline(mode string, args []string) error {
return err
}
var signals *telemetry.Signals
if mode == "worker" {
if mode == "worker" || mode == "dispatcher" {
service := "messenger-" + mode
var stopTelemetry func()
signals, stopTelemetry, err = telemetry.Start("messenger-worker")
signals, stopTelemetry, err = telemetry.Start(service)
if err != nil {
return err
}
defer stopTelemetry()
if err := signals.ObservePool(db); err != nil {
return err
}
if mode == "dispatcher" {
if err := signals.ObserveBacklog(store.PendingJobs); err != nil {
return err
}
}
}
for ctx.Err() == nil {
broker, e := repositories.NewJobBroker(ctx, address)
if e == nil {
if mode == "dispatcher" {
e = dispatchJobs(ctx, store, broker)
e = dispatchJobs(ctx, store, broker, signals)
} else {
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals)
}
@@ -89,7 +100,7 @@ func RunPipeline(mode string, args []string) error {
return nil
}
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error {
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker, signals *telemetry.Signals) error {
for ctx.Err() == nil {
jobs, err := store.QueuedJobs(ctx)
if err != nil {
@@ -102,7 +113,12 @@ func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, brok
return err
}
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
err = broker.Publish(publishCtx, payload)
publishCtx = telemetry.WithTraceHeaders(publishCtx, map[string]string{
"traceparent": job.Traceparent, "tracestate": job.Tracestate,
})
err = signals.Publish(publishCtx, func(ctx context.Context) error {
return broker.Publish(ctx, payload)
})
cancel()
if err != nil {
return err
@@ -142,14 +158,19 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
continue
}
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
task, finish := signals.Job(task, job.Type, job.AttachmentID)
parent, _ := message.Headers["traceparent"].(string)
state, _ := message.Headers["tracestate"].(string)
task = telemetry.WithTraceHeaders(task, map[string]string{"traceparent": parent, "tracestate": state})
task, finish := signals.Job(task, job.Type, job.AttachmentID, job.Attempt, job.CreatedAt)
var imageErr error
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
if workErr == nil && token != "" {
faults.Crash("before_effect", job.AttachmentID)
faults.Pause(task, "after_claim", job.AttachmentID)
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
variants, code, err := processImage(task, objects, a, staging)
imageCtx, finishImage := telemetry.Operation(task, "s3 read staging and write variants", trace.SpanKindClient)
variants, code, err := processImage(imageCtx, objects, a, staging)
finishImage(err)
imageErr = err
return variants, code, err
})