forked from mtusi2026/mtusi2026lab6
feat: add pipeline tracing and observability alerts
This commit is contained in:
+2
-1
@@ -203,7 +203,7 @@ services:
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
networks: [data]
|
||||
networks: [data, telemetry]
|
||||
environment: *api-environment
|
||||
volumes: *api-volumes
|
||||
command: ["./messenger", "dispatcher"]
|
||||
@@ -334,6 +334,7 @@ services:
|
||||
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
|
||||
volumes:
|
||||
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
|
||||
- ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
|
||||
- prometheus_data:/prometheus
|
||||
|
||||
loki:
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
groups:
|
||||
- name: messenger
|
||||
interval: 5s
|
||||
rules:
|
||||
- alert: MessengerHighErrorRate
|
||||
expr: (sum(rate(messenger_http_requests_total{http_status_class="5xx"}[1m])) / sum(rate(messenger_http_requests_total[1m]))) > 0.05
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: More than 5% of business requests return 5xx over one minute
|
||||
runbook: Check API error spans and PostgreSQL operations in Tempo.
|
||||
- alert: MessengerQueueLag
|
||||
expr: max(messenger_queue_oldest_seconds) > 20
|
||||
for: 15s
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Oldest outstanding attachment job is over 20 seconds old
|
||||
runbook: Check outstanding jobs, worker saturation and consumer traces.
|
||||
@@ -200,6 +200,314 @@
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"title": "API · p50 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"title": "API · p99 длительности",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 24,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"title": "API · доля 5xx",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "(sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))) / sum(rate(messenger_http_requests_total[1m]))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"title": "Очередь attachments.v1 · незавершённые задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 32,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_depth)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"title": "Очередь attachments.v1 · возраст старейшего задания",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "max(messenger_queue_oldest_seconds)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"title": "PostgreSQL · занятые соединения",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 40,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_db_pool_used)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"title": "PostgreSQL · запросы к пулу, ожидавшие соединение, в секунду",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_db_pool_waited_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"title": "Worker · загрузка",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 48,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum(messenger_worker_busy) / sum(messenger_worker_capacity)",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "percentunit"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"title": "Вложения · p95 до ready",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_attachment_ready_seconds_bucket[1m])))",
|
||||
"legendFormat": ""
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "s"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"title": "Процессы · загрузка CPU",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 12,
|
||||
"y": 56,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "cores"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"title": "Процессы · резидентная память",
|
||||
"type": "timeseries",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "lab6-prometheus"
|
||||
},
|
||||
"gridPos": {
|
||||
"x": 0,
|
||||
"y": 64,
|
||||
"w": 12,
|
||||
"h": 8
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"refId": "A",
|
||||
"expr": "sum by (service_name) (messenger_process_rss_bytes)",
|
||||
"legendFormat": "{{service_name}}"
|
||||
}
|
||||
],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "bytes"
|
||||
},
|
||||
"overrides": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
global:
|
||||
scrape_interval: 5s
|
||||
rule_files:
|
||||
- /etc/prometheus/alerts.yaml
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# Диагностика API и обработки вложений
|
||||
|
||||
SLI доступности API — доля бизнес-запросов без 5xx. Цель — ≥99% за 30 минут при наличии трафика. Проверка:
|
||||
|
||||
```promql
|
||||
1 - (sum(increase(messenger_http_requests_total{http_status_class="5xx"}[30m])) or 0 * sum(increase(messenger_http_requests_total[30m]))) / sum(increase(messenger_http_requests_total[30m]))
|
||||
```
|
||||
|
||||
Health исключён из обоих счётчиков. `MessengerHighErrorRate` срабатывает при доле 5xx выше 5% за минуту в течение 15 секунд. В Explore → Tempo найти ошибочный серверный спан и дочерний `postgresql query`. Сравнить длительность дочернего спана с длительностью HTTP, затем сверить занятые соединения и rate запросов, ожидавших пул.
|
||||
|
||||
`MessengerQueueLag` срабатывает, когда старейшее незавершённое задание старше 20 секунд в течение 15 секунд. В dashboard проверить число незавершённых заданий и наличие worker. Если возраст растёт при отсутствии новых consumer spans, проверить процесс worker. После восстановления дождаться перехода заданий в ready и исчезновения активного alert.
|
||||
|
||||
Глубина очереди измеряется по устойчивым заданиям в PostgreSQL. В неё входят работа в RabbitMQ, leases и ожидание retry. Возраст измеряется от создания задания. Время до ready включает ожидание и повторные попытки. Гистограммы времени используют секунды. Загрузка worker — доля занятых слотов, по одному слоту на процесс. Ожидание пула отображается как частота успешных получений соединения после ожидания. Метрики CPU и RSS относятся к основным процессам Go.
|
||||
|
||||
Правила доступны через API Prometheus `/api/v1/alerts` или запрос `ALERTS{alertstate="firing"}` в Grafana. При недоступности Loki или Tempo приложение продолжает работу. Collector ограничивает память и очередь экспорта, повторяет отправку до 30 секунд. Сигналы сверх лимитов или после истечения retry могут быть потеряны.
|
||||
|
||||
Команды локальной проверки после `make app-up`:
|
||||
|
||||
```sh
|
||||
python3 scripts/check_observability.py pipeline
|
||||
python3 scripts/check_observability_incidents.py worker
|
||||
python3 scripts/check_observability_incidents.py database
|
||||
python3 scripts/check_observability_incidents.py retry
|
||||
python3 scripts/check_observability_incidents.py telemetry
|
||||
```
|
||||
|
||||
Сценарии выполняются последовательно на учебном стенде. DB-инцидент держит временную блокировку таблицы чатов. Проверка retry кратко останавливает S3 и проверяет номера попыток в одной трассе исходного upload. Проверка telemetry останавливает Tempo и одновременно отправляет сообщения и обрабатывает вложение.
|
||||
@@ -31,7 +31,7 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals"])
|
||||
parser.add_argument("criterion", choices=["provisioning", "signals", "pipeline"])
|
||||
args = parser.parse_args()
|
||||
|
||||
def query(path, **params):
|
||||
@@ -47,13 +47,42 @@ def main():
|
||||
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
|
||||
print("Источник Grafana:", uid)
|
||||
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
|
||||
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
|
||||
check(len(dashboard["panels"]) >= 6, "dashboard panels missing")
|
||||
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
|
||||
return
|
||||
|
||||
client = Client(args.base_url)
|
||||
start = time.time_ns()
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
if args.criterion == "pipeline":
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
client.wait_ready(bob, upload["json"])
|
||||
request_id = upload["headers"]["X-Request-Id"]
|
||||
deadline = time.monotonic() + 45
|
||||
while time.monotonic() < deadline:
|
||||
logs = source("loki", "/loki/api/v1/query_range",
|
||||
query='{service_name="messenger-api"} | request_id="' + request_id + '"',
|
||||
start=start, end=time.time_ns())["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
trace = source("tempo", "/api/traces/" + trace_id)
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
producer = next((span for span in spans if span["kind"] == "SPAN_KIND_PRODUCER"), None)
|
||||
consumer = next((span for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"), None)
|
||||
if producer and consumer:
|
||||
api = next(span for span in spans if span["kind"] == "SPAN_KIND_SERVER")
|
||||
check(producer["parentSpanId"] == api["spanId"], "producer parent mismatch")
|
||||
check(consumer["parentSpanId"] == producer["spanId"], "consumer parent mismatch")
|
||||
children = {span["name"] for span in spans if span.get("parentSpanId") == consumer["spanId"]}
|
||||
check({"postgresql finalize attachment", "s3 read staging and write variants"} <= children,
|
||||
"worker S3 and PostgreSQL spans missing")
|
||||
print("Upload → producer → consumer → S3/PostgreSQL, trace:", trace_id)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure("upload trace did not include the worker pipeline")
|
||||
|
||||
marker = "telemetry-secret-" + uuid.uuid4().hex
|
||||
message = client.message(alice, chat, marker)
|
||||
request_id = message["headers"]["X-Request-Id"]
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Локальные проверки отказов уровня 4. Результаты выводятся в терминал."""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, timeout=90)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("incident", choices=["worker", "database", "telemetry", "retry"])
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
def trace_for(request_id, matches):
|
||||
base = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-"
|
||||
query = urllib.parse.urlencode({"query": '{service_name="messenger-api"} | request_id="' + request_id + '"'})
|
||||
deadline = time.monotonic() + 35
|
||||
while time.monotonic() < deadline:
|
||||
with urllib.request.urlopen(base + "loki/loki/api/v1/query_range?" + query, timeout=5) as response:
|
||||
logs = json.load(response)["data"]["result"]
|
||||
if logs:
|
||||
trace_id = logs[0]["stream"]["trace_id"]
|
||||
try:
|
||||
with urllib.request.urlopen(base + "tempo/api/traces/" + trace_id, timeout=5) as response:
|
||||
trace = json.load(response)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code != 404:
|
||||
raise
|
||||
else:
|
||||
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
|
||||
if matches(spans):
|
||||
return trace_id
|
||||
time.sleep(2)
|
||||
raise Failure("expected trace was not found")
|
||||
|
||||
def alerts(name):
|
||||
endpoint = args.grafana_url.rstrip("/") + "/api/datasources/proxy/uid/lab6-prometheus/api/v1/alerts"
|
||||
with urllib.request.urlopen(endpoint, timeout=5) as response:
|
||||
active = json.load(response)["data"]["alerts"]
|
||||
return any(item["labels"]["alertname"] == name and item["state"] == "firing" for item in active)
|
||||
|
||||
def wait_alert(name, firing, exercise=lambda: None):
|
||||
deadline = time.monotonic() + 100
|
||||
while time.monotonic() < deadline:
|
||||
exercise()
|
||||
if alerts(name) == firing:
|
||||
print(name, "firing" if firing else "resolved", flush=True)
|
||||
return
|
||||
time.sleep(2)
|
||||
raise Failure(name + ": expected alert state not observed")
|
||||
|
||||
if args.incident == "worker":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
wait_alert("MessengerQueueLag", True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
client.wait_ready(alice, attachment)
|
||||
wait_alert("MessengerQueueLag", False)
|
||||
elif args.incident == "database":
|
||||
# Hold a read-blocking lock on chats. The health probe can still reach PostgreSQL.
|
||||
marker = "lab6-fault-" + uuid.uuid4().hex
|
||||
command = ["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" '
|
||||
'-c "BEGIN; LOCK TABLE chats IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(75); ROLLBACK; /* $1 */"',
|
||||
"sh", marker]
|
||||
lock = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.DEVNULL)
|
||||
try:
|
||||
time.sleep(2)
|
||||
|
||||
request_ids = []
|
||||
|
||||
def exercise():
|
||||
response = client.call("GET", "/api/v1/chats/" + chat["id"], [500, 503], actor=alice)
|
||||
request_ids.append(response["headers"]["X-Request-Id"])
|
||||
|
||||
wait_alert("MessengerHighErrorRate", True, exercise)
|
||||
finally:
|
||||
# Terminate only the synthetic lock session so the transaction rolls back immediately.
|
||||
compose("exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh",
|
||||
"SELECT pg_terminate_backend(pid) FROM pg_stat_activity "
|
||||
"WHERE pid<>pg_backend_pid() AND query LIKE '%/* " + marker + " */'")
|
||||
lock.wait(timeout=10)
|
||||
client.call("GET", "/api/v1/chats/" + chat["id"], actor=alice, schema="Chat")
|
||||
trace_id = trace_for(request_ids[-1], lambda spans: any(
|
||||
span["name"] == "postgresql query"
|
||||
and int(span["endTimeUnixNano"]) - int(span["startTimeUnixNano"]) > 1_000_000_000 for span in spans))
|
||||
print("Медленный запрос PostgreSQL подтверждён в trace:", trace_id, flush=True)
|
||||
wait_alert("MessengerHighErrorRate", False)
|
||||
elif args.incident == "retry":
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
upload = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
|
||||
202, actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(),
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
attachment = upload["json"]
|
||||
compose("stop", "s3")
|
||||
try:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 25
|
||||
while time.monotonic() < deadline:
|
||||
sql = ("SELECT count(*) FROM attachment_job_attempts a JOIN attachment_jobs j ON j.id=a.job_id "
|
||||
"WHERE a.next_attempt_at IS NOT NULL AND j.attachment_id='" + str(uuid.UUID(attachment["id"])) + "'")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'exec psql -X -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', "sh", sql],
|
||||
cwd=ROOT, text=True, timeout=10)
|
||||
if int(output.strip()) > 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise Failure("durable retry was not observed")
|
||||
finally:
|
||||
compose("up", "-d", "--wait", "s3")
|
||||
client.wait_ready(alice, attachment)
|
||||
def retried(spans):
|
||||
attempts = [a["value"]["intValue"] for span in spans if span["kind"] == "SPAN_KIND_CONSUMER"
|
||||
for a in span.get("attributes", []) if a["key"] == "job.attempt"]
|
||||
return len(set(attempts)) >= 2
|
||||
|
||||
trace_id = trace_for(upload["headers"]["X-Request-Id"], retried)
|
||||
print("Исходное вложение перешло в ready, обе попытки в trace:", trace_id, flush=True)
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
else:
|
||||
since = str(int(time.time()))
|
||||
compose("stop", "tempo")
|
||||
try:
|
||||
for _ in range(8):
|
||||
client.message(alice, chat, "telemetry-backend-off")
|
||||
time.sleep(5)
|
||||
attachment, _ = client.attachment(alice, chat)
|
||||
client.wait_ready(alice, attachment)
|
||||
compose("stats", "--no-stream", "otel-collector")
|
||||
print("API и worker работали при остановленном Tempo.", flush=True)
|
||||
finally:
|
||||
compose("start", "tempo")
|
||||
output = subprocess.check_output(
|
||||
["docker", "compose", "logs", "--no-color", "--since", since, "otel-collector"],
|
||||
cwd=ROOT, text=True, timeout=15)
|
||||
dropped = sum(int(match) for line in output.splitlines() if '"otlphttp/tempo"' in line
|
||||
for match in re.findall(r'"dropped_items":\s*(\d+)', line))
|
||||
print("Спанов отброшено Collector за период проверки:", dropped, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, OSError, subprocess.SubprocessError, ValueError) as error:
|
||||
print("FAIL:", error, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
@@ -94,6 +94,9 @@ func RunHTTP() error {
|
||||
return fmt.Errorf("telemetry initialization failed: %w", err)
|
||||
}
|
||||
defer stopTelemetry()
|
||||
if err := signals.ObservePool(db); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
server := &http.Server{
|
||||
Addr: serverAddress(),
|
||||
|
||||
@@ -2,7 +2,9 @@ package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"errors"
|
||||
"messenger/src/backend/telemetry"
|
||||
"net"
|
||||
"time"
|
||||
|
||||
@@ -53,8 +55,12 @@ func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
|
||||
|
||||
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
// mandatory catches an unroutable message; confirms alone do not prove routing.
|
||||
headers := amqp.Table{}
|
||||
for key, value := range telemetry.TraceHeaders(ctx) {
|
||||
headers[key] = value
|
||||
}
|
||||
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
|
||||
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body,
|
||||
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, Headers: headers,
|
||||
})
|
||||
if err != nil {
|
||||
return errors.New("RabbitMQ publication failed")
|
||||
|
||||
@@ -10,6 +10,8 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/telemetry"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
pgxmigrate "github.com/golang-migrate/migrate/v4/database/pgx/v5"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
@@ -43,6 +45,7 @@ func OpenDatabase(ctx context.Context) (*pgxpool.Pool, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cfg.ConnConfig.Tracer = telemetry.PostgresTracer{}
|
||||
cfg.MaxConns = 10
|
||||
if value := os.Getenv("DATABASE_MAX_CONNS"); value != "" {
|
||||
maxConns, parseErr := strconv.Atoi(value)
|
||||
|
||||
@@ -5,10 +5,14 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"math/big"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/telemetry"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type Variant struct {
|
||||
@@ -27,12 +31,14 @@ type Job struct {
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
Traceparent string `json:"-"`
|
||||
Tracestate string `json:"-"`
|
||||
}
|
||||
|
||||
// QueuedJobs is also the recovery source when publish/confirm is uncertain.
|
||||
// Re-publication is intentional: a durable intent is never removed by publish.
|
||||
func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,j.pipeline_version,j.attempts+1,j.created_at,j.traceparent,j.tracestate FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing') AND j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) AND (j.published_at IS NULL OR j.published_at<now()-interval '30 seconds') ORDER BY j.created_at LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -40,7 +46,7 @@ func (s *AttachmentStore) QueuedJobs(ctx context.Context) ([]Job, error) {
|
||||
jobs := []Job{}
|
||||
for rows.Next() {
|
||||
j := Job{SchemaVersion: 1, Type: "attachment.process.v1"}
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt); err != nil {
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.CreatedAt, &j.Traceparent, &j.Tracestate); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
j.CreatedAt = j.CreatedAt.UTC()
|
||||
@@ -104,7 +110,9 @@ var ErrProcessingTimeout = errors.New("processing timeout")
|
||||
|
||||
// ProcessJob holds the row lock across deterministic object writes. The lock and
|
||||
// token check prevent overlapping/stale attempts from publishing metadata.
|
||||
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
|
||||
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) (result error) {
|
||||
ctx, finish := telemetry.Operation(ctx, "postgresql finalize attachment", trace.SpanKindClient)
|
||||
defer func() { finish(result) }()
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -202,7 +210,8 @@ func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) er
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, uuid.NewString(), a.ID, staging)
|
||||
headers := telemetry.TraceHeaders(ctx)
|
||||
_, err = tx.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key,traceparent,tracestate) VALUES($1,$2,$3,$4,$5)`, uuid.NewString(), a.ID, staging, headers["traceparent"], headers["tracestate"])
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -211,3 +220,13 @@ func (s *AttachmentStore) JobTerminal(ctx context.Context, id string) (bool, err
|
||||
err := s.db.QueryRow(ctx, `SELECT j.superseded_at IS NOT NULL OR j.pipeline_version<a.pipeline_version OR a.status IN ('ready','failed') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, id).Scan(&done)
|
||||
return done, err
|
||||
}
|
||||
|
||||
// PendingJobs counts durable outstanding work, including leases and scheduled retries.
|
||||
func (s *AttachmentStore) PendingJobs(ctx context.Context) (int64, float64, error) {
|
||||
var count int64
|
||||
var age float64
|
||||
err := s.db.QueryRow(ctx, `SELECT count(*), COALESCE(EXTRACT(EPOCH FROM now()-min(j.created_at)),0)::double precision
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
||||
WHERE j.superseded_at IS NULL AND j.pipeline_version=a.pipeline_version AND a.status IN ('queued','processing')`).Scan(&count, &age)
|
||||
return count, age, err
|
||||
}
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE attachment_jobs DROP COLUMN tracestate, DROP COLUMN traceparent;
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_jobs
|
||||
ADD COLUMN traceparent text NOT NULL DEFAULT '',
|
||||
ADD COLUMN tracestate text NOT NULL DEFAULT '';
|
||||
@@ -0,0 +1,94 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"go.opentelemetry.io/otel/metric"
|
||||
)
|
||||
|
||||
func (s *Signals) ObservePool(pool *pgxpool.Pool) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
used, err := s.meter.Int64ObservableGauge("messenger.db.pool.used")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
capacity, err := s.meter.Int64ObservableGauge("messenger.db.pool.capacity")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
waited, err := s.meter.Int64ObservableCounter("messenger.db.pool.waited")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
|
||||
stats := pool.Stat()
|
||||
observer.ObserveInt64(used, int64(stats.AcquiredConns()))
|
||||
observer.ObserveInt64(capacity, int64(stats.MaxConns()))
|
||||
observer.ObserveInt64(waited, stats.EmptyAcquireCount())
|
||||
return nil
|
||||
}, used, capacity, waited)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Signals) ObserveBacklog(read func(context.Context) (int64, float64, error)) error {
|
||||
if s == nil {
|
||||
return nil
|
||||
}
|
||||
depth, err := s.meter.Int64ObservableGauge("messenger.queue.depth")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
age, err := s.meter.Float64ObservableGauge("messenger.queue.oldest", metric.WithUnit("s"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.meter.RegisterCallback(func(ctx context.Context, observer metric.Observer) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, time.Second)
|
||||
defer cancel()
|
||||
count, seconds, err := read(ctx)
|
||||
if err == nil {
|
||||
queue := metric.WithAttributes(attribute.String("queue.name", "attachments.v1"))
|
||||
observer.ObserveInt64(depth, count, queue)
|
||||
observer.ObserveFloat64(age, seconds, queue)
|
||||
}
|
||||
return nil
|
||||
}, depth, age)
|
||||
return err
|
||||
}
|
||||
|
||||
// ObserveProcess exports Linux process CPU time and resident memory.
|
||||
func (s *Signals) ObserveProcess() error {
|
||||
cpu, err := s.meter.Float64ObservableCounter("messenger.process.cpu", metric.WithUnit("s"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rss, err := s.meter.Int64ObservableGauge("messenger.process.rss", metric.WithUnit("By"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = s.meter.RegisterCallback(func(_ context.Context, observer metric.Observer) error {
|
||||
var usage syscall.Rusage
|
||||
if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
|
||||
observer.ObserveFloat64(cpu, float64(usage.Utime.Nano()+usage.Stime.Nano())/float64(time.Second))
|
||||
}
|
||||
if data, err := os.ReadFile("/proc/self/statm"); err == nil {
|
||||
fields := strings.Fields(string(data))
|
||||
if len(fields) >= 2 {
|
||||
if pages, err := strconv.ParseInt(fields[1], 10, 64); err == nil {
|
||||
observer.ObserveInt64(rss, pages*int64(os.Getpagesize()))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}, cpu, rss)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
type PostgresTracer struct{}
|
||||
|
||||
type queryFinishKey struct{}
|
||||
|
||||
func (PostgresTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
|
||||
ctx, finish := Operation(ctx, "postgresql query", trace.SpanKindClient)
|
||||
return context.WithValue(ctx, queryFinishKey{}, finish)
|
||||
}
|
||||
|
||||
func (PostgresTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) {
|
||||
if finish, ok := ctx.Value(queryFinishKey{}).(func(error)); ok {
|
||||
finish(data.Err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package telemetry
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/propagation"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func TraceHeaders(ctx context.Context) map[string]string {
|
||||
headers := propagation.MapCarrier{}
|
||||
propagation.TraceContext{}.Inject(ctx, headers)
|
||||
return headers
|
||||
}
|
||||
|
||||
func WithTraceHeaders(ctx context.Context, headers map[string]string) context.Context {
|
||||
return propagation.TraceContext{}.Extract(ctx, propagation.MapCarrier(headers))
|
||||
}
|
||||
|
||||
func (s *Signals) Publish(ctx context.Context, publish func(context.Context) error) error {
|
||||
if s == nil {
|
||||
return publish(ctx)
|
||||
}
|
||||
ctx, span := s.tracer.Start(ctx, "attachments.v1 publish", trace.WithSpanKind(trace.SpanKindProducer))
|
||||
defer span.End()
|
||||
err := publish(ctx)
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, "")
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Operation creates a child of the active span without recording arguments or raw errors.
|
||||
func Operation(ctx context.Context, name string, kind trace.SpanKind) (context.Context, func(error)) {
|
||||
parent := trace.SpanFromContext(ctx)
|
||||
if !parent.IsRecording() {
|
||||
return ctx, func(error) {}
|
||||
}
|
||||
ctx, span := parent.TracerProvider().Tracer("messenger").Start(ctx, name, trace.WithSpanKind(kind))
|
||||
return ctx, func(err error) {
|
||||
if err != nil {
|
||||
span.SetStatus(codes.Error, "")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,9 @@ import (
|
||||
)
|
||||
|
||||
type Signals struct {
|
||||
meter metric.Meter
|
||||
busy metric.Int64UpDownCounter
|
||||
ready metric.Float64Histogram
|
||||
tracer trace.Tracer
|
||||
logger otellog.Logger
|
||||
requests metric.Int64Counter
|
||||
@@ -79,11 +82,25 @@ func Start(service string) (*Signals, func(), error) {
|
||||
httpDuration, errHTTP := meter.Float64Histogram("messenger.http.duration", metric.WithUnit("s"), buckets)
|
||||
jobs, errJobs := meter.Int64Counter("messenger.worker.jobs")
|
||||
jobDuration, errJobDuration := meter.Float64Histogram("messenger.worker.duration", metric.WithUnit("s"), buckets)
|
||||
if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration); err != nil {
|
||||
busy, errBusy := meter.Int64UpDownCounter("messenger.worker.busy")
|
||||
capacity, errCapacity := meter.Int64UpDownCounter("messenger.worker.capacity")
|
||||
ready, errReady := meter.Float64Histogram("messenger.attachment.ready", metric.WithUnit("s"), buckets)
|
||||
if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration, errBusy, errCapacity, errReady); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return &Signals{tp.Tracer("messenger"), lp.Logger("messenger"), requests, httpDuration, jobs, jobDuration}, stop, nil
|
||||
if service == "messenger-worker" {
|
||||
capacity.Add(ctx, 1)
|
||||
busy.Add(ctx, 0)
|
||||
}
|
||||
signals := &Signals{meter: meter, busy: busy, ready: ready, tracer: tp.Tracer("messenger"),
|
||||
logger: lp.Logger("messenger"), requests: requests, httpDuration: httpDuration,
|
||||
jobs: jobs, jobDuration: jobDuration}
|
||||
if err := signals.ObserveProcess(); err != nil {
|
||||
stop()
|
||||
return nil, func() {}, err
|
||||
}
|
||||
return signals, stop, nil
|
||||
}
|
||||
|
||||
func (s *Signals) event(ctx context.Context, event string, failed bool, attrs ...otellog.KeyValue) {
|
||||
|
||||
@@ -11,14 +11,19 @@ import (
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
func (s *Signals) Job(ctx context.Context, jobType, attachmentID string) (context.Context, func(string)) {
|
||||
func (s *Signals) Job(ctx context.Context, jobType, attachmentID string, attempt int, accepted time.Time) (context.Context, func(string)) {
|
||||
if s == nil {
|
||||
return ctx, func(string) {}
|
||||
}
|
||||
start := time.Now()
|
||||
ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer))
|
||||
s.busy.Add(ctx, 1)
|
||||
ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer), trace.WithAttributes(attribute.Int("job.attempt", attempt)))
|
||||
return ctx, func(outcome string) {
|
||||
defer span.End()
|
||||
s.busy.Add(ctx, -1)
|
||||
if outcome == "success" {
|
||||
s.ready.Record(ctx, time.Since(accepted).Seconds())
|
||||
}
|
||||
attrs := []attribute.KeyValue{attribute.String("job.type", jobType), attribute.String("job.outcome", outcome)}
|
||||
span.SetAttributes(attrs...)
|
||||
failed := outcome == "error"
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"messenger/src/backend/internal/faults"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
@@ -60,19 +62,28 @@ func RunPipeline(mode string, args []string) error {
|
||||
return err
|
||||
}
|
||||
var signals *telemetry.Signals
|
||||
if mode == "worker" {
|
||||
if mode == "worker" || mode == "dispatcher" {
|
||||
service := "messenger-" + mode
|
||||
var stopTelemetry func()
|
||||
signals, stopTelemetry, err = telemetry.Start("messenger-worker")
|
||||
signals, stopTelemetry, err = telemetry.Start(service)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stopTelemetry()
|
||||
if err := signals.ObservePool(db); err != nil {
|
||||
return err
|
||||
}
|
||||
if mode == "dispatcher" {
|
||||
if err := signals.ObserveBacklog(store.PendingJobs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
broker, e := repositories.NewJobBroker(ctx, address)
|
||||
if e == nil {
|
||||
if mode == "dispatcher" {
|
||||
e = dispatchJobs(ctx, store, broker)
|
||||
e = dispatchJobs(ctx, store, broker, signals)
|
||||
} else {
|
||||
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals)
|
||||
}
|
||||
@@ -89,7 +100,7 @@ func RunPipeline(mode string, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error {
|
||||
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker, signals *telemetry.Signals) error {
|
||||
for ctx.Err() == nil {
|
||||
jobs, err := store.QueuedJobs(ctx)
|
||||
if err != nil {
|
||||
@@ -102,7 +113,12 @@ func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, brok
|
||||
return err
|
||||
}
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.Publish(publishCtx, payload)
|
||||
publishCtx = telemetry.WithTraceHeaders(publishCtx, map[string]string{
|
||||
"traceparent": job.Traceparent, "tracestate": job.Tracestate,
|
||||
})
|
||||
err = signals.Publish(publishCtx, func(ctx context.Context) error {
|
||||
return broker.Publish(ctx, payload)
|
||||
})
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -142,14 +158,19 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
|
||||
continue
|
||||
}
|
||||
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
|
||||
task, finish := signals.Job(task, job.Type, job.AttachmentID)
|
||||
parent, _ := message.Headers["traceparent"].(string)
|
||||
state, _ := message.Headers["tracestate"].(string)
|
||||
task = telemetry.WithTraceHeaders(task, map[string]string{"traceparent": parent, "tracestate": state})
|
||||
task, finish := signals.Job(task, job.Type, job.AttachmentID, job.Attempt, job.CreatedAt)
|
||||
var imageErr error
|
||||
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
|
||||
if workErr == nil && token != "" {
|
||||
faults.Crash("before_effect", job.AttachmentID)
|
||||
faults.Pause(task, "after_claim", job.AttachmentID)
|
||||
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
variants, code, err := processImage(task, objects, a, staging)
|
||||
imageCtx, finishImage := telemetry.Operation(task, "s3 read staging and write variants", trace.SpanKindClient)
|
||||
variants, code, err := processImage(imageCtx, objects, a, staging)
|
||||
finishImage(err)
|
||||
imageErr = err
|
||||
return variants, code, err
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user