feat: add baseline observability for lab 6

This commit is contained in:
Dmitrii Grishko
2026-11-13 18:24:14 +03:00
parent d993866881
commit 549edc8d7a
17 changed files with 967 additions and 33 deletions
+54 -2
View File
@@ -102,13 +102,14 @@ services:
api:
cpus: 0.5
mem_limit: 512m
networks: [proxy, data]
networks: [proxy, data, telemetry]
build: ./src
deploy:
replicas: ${API_REPLICAS:-2}
stop_grace_period: 15s
environment: &api-environment
<<: *s3-environment
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT-http://otel-collector:4318}
S3_ROOT_ACCESS_KEY_ID_FILE: ""
S3_ROOT_SECRET_ACCESS_KEY_FILE: ""
BROKER_URL_FILE: /run/secrets/broker_url
@@ -184,7 +185,7 @@ services:
restart: unless-stopped
cpus: 1.0
mem_limit: 512m
networks: [data]
networks: [data, telemetry]
environment: *api-environment
volumes: *api-volumes
command: ["./messenger", "worker"]
@@ -319,7 +320,56 @@ services:
timeout: 3s
retries: 20
otel-collector:
image: otel/opentelemetry-collector-contrib:0.120.0
networks: [telemetry]
mem_limit: 192m
command: ["--config=/etc/otelcol-contrib/config.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/config.yaml:ro
prometheus:
image: prom/prometheus:v3.2.1
networks: [telemetry]
command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.path=/prometheus", "--storage.tsdb.retention.time=7d"]
volumes:
- ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
loki:
image: grafana/loki:3.4.2
networks: [telemetry]
command: ["-config.file=/etc/loki/lab.yaml"]
volumes:
- ./infra/otel/loki.yaml:/etc/loki/lab.yaml:ro
- loki_data:/loki
tempo:
image: grafana/tempo:2.7.1
networks: [telemetry]
command: ["-config.file=/etc/tempo/lab.yaml"]
volumes:
- ./infra/otel/tempo.yaml:/etc/tempo/lab.yaml:ro
- tempo_data:/var/tempo
grafana:
image: grafana/grafana:11.5.2
networks: [telemetry, edge]
environment:
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
ports:
- "127.0.0.1:${GRAFANA_HTTP_PORT:-3000}:3000"
volumes:
- ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
volumes:
prometheus_data:
loki_data:
tempo_data:
grafana_data:
rabbitmq_data:
postgres_data:
redis_data:
@@ -331,3 +381,5 @@ networks:
internal: true
data:
internal: true
telemetry:
internal: true
+45
View File
@@ -0,0 +1,45 @@
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
processors:
memory_limiter:
check_interval: 1s
limit_mib: 128
spike_limit_mib: 32
batch:
timeout: 1s
send_batch_size: 256
send_batch_max_size: 512
exporters:
prometheus:
endpoint: 0.0.0.0:8889
resource_to_telemetry_conversion:
enabled: true
otlphttp/loki:
endpoint: http://loki:3100/otlp
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
otlphttp/tempo:
endpoint: http://tempo:4318
sending_queue:
queue_size: 64
retry_on_failure:
max_elapsed_time: 30s
service:
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
@@ -0,0 +1,205 @@
{
"uid": "messenger-lab6",
"title": "Мессенджер · API и worker",
"schemaVersion": 39,
"refresh": "5s",
"time": {
"from": "now-15m",
"to": "now"
},
"panels": [
{
"id": 1,
"title": "API · запросы в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 2,
"title": "API · 5xx в секунду",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "sum(rate(messenger_http_requests_total{http_status_class=\"5xx\"}[1m])) or 0 * sum(rate(messenger_http_requests_total[1m]))",
"legendFormat": ""
}
],
"gridPos": {
"x": 12,
"y": 0,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "reqps"
},
"overrides": []
}
},
{
"id": 3,
"title": "API · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket[1m])))",
"legendFormat": ""
}
],
"gridPos": {
"x": 0,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 4,
"title": "Worker · задания в минуту",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "60 * sum by (job_outcome) (rate(messenger_worker_jobs_total[1m]))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 12,
"y": 8,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "short"
},
"overrides": []
}
},
{
"id": 5,
"title": "Worker · p95 длительности",
"type": "timeseries",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "prometheus",
"uid": "lab6-prometheus"
},
"expr": "histogram_quantile(0.95, sum by (le,job_outcome) (rate(messenger_worker_duration_seconds_bucket[1m])))",
"legendFormat": "{{job_outcome}}"
}
],
"gridPos": {
"x": 0,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": "s"
},
"overrides": []
}
},
{
"id": 6,
"title": "API и worker · события",
"type": "logs",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"targets": [
{
"refId": "A",
"datasource": {
"type": "loki",
"uid": "lab6-loki"
},
"expr": "{service_name=~\"messenger-api|messenger-worker\"}",
"legendFormat": "",
"queryType": "range"
}
],
"gridPos": {
"x": 12,
"y": 16,
"w": 12,
"h": 8
},
"fieldConfig": {
"defaults": {
"unit": ""
},
"overrides": []
}
}
]
}
@@ -0,0 +1,6 @@
apiVersion: 1
providers:
- name: Messenger
type: file
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,32 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: lab6-prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
- name: Loki
uid: lab6-loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: Trace
matcherType: label
matcherRegex: trace_id
datasourceUid: lab6-tempo
url: '$${__value.raw}'
- name: Tempo
uid: lab6-tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: lab6-loki
spanStartTimeShift: '-5s'
spanEndTimeShift: '5s'
customQuery: true
query: '{service_name=~"messenger-api|messenger-worker"} | trace_id="$${__span.traceId}"'
+31
View File
@@ -0,0 +1,31 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
replication_factor: 1
ring:
instance_addr: 127.0.0.1
kvstore:
store: inmemory
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
schema_config:
configs:
- from: 2024-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
otlp_config:
resource_attributes:
ignore_defaults: true
attributes_config:
- action: index_label
attributes: [service.name, deployment.environment.name]
+6
View File
@@ -0,0 +1,6 @@
global:
scrape_interval: 5s
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: [otel-collector:8889]
+18
View File
@@ -0,0 +1,18 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
compactor:
compaction:
block_retention: 48h
+151
View File
@@ -0,0 +1,151 @@
#!/usr/bin/env python3
"""Проверка сохранённых сигналов через источники Grafana."""
import argparse
import base64
import json
from pathlib import Path
import re
import struct
import sys
import time
import urllib.parse
import urllib.request
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check
def oversized_image():
# 9 million pixels fit the API limit but exceed the default worker limit.
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
header = struct.pack(">IIBBBBB", 3000, 3000, 8, 0, 0, 0, 0)
return (bytes.fromhex("89504e470d0a1a0a") + chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(bytes(3001 * 3000))) + chunk(b"IEND", b""))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
parser.add_argument("criterion", choices=["provisioning", "signals"])
args = parser.parse_args()
def query(path, **params):
suffix = "?" + urllib.parse.urlencode(params) if params else ""
with urllib.request.urlopen(args.grafana_url.rstrip("/") + path + suffix, timeout=5) as response:
return json.load(response)
def source(uid, path, **params):
return query("/api/datasources/proxy/uid/lab6-" + uid + path, **params)
if args.criterion == "provisioning":
for uid in ("prometheus", "loki", "tempo"):
check(query("/api/datasources/uid/lab6-" + uid)["type"] == uid, "source type mismatch")
print("Источник Grafana:", uid)
dashboard = query("/api/dashboards/uid/messenger-lab6")["dashboard"]
check(len(dashboard["panels"]) == 6, "dashboard panels missing")
print("Загружен dashboard messenger-lab6, панелей:", len(dashboard["panels"]))
return
client = Client(args.base_url)
start = time.time_ns()
alice, bob, _, chat = client.bootstrap()
marker = "telemetry-secret-" + uuid.uuid4().hex
message = client.message(alice, chat, marker)
request_id = message["headers"]["X-Request-Id"]
attachment, original = client.attachment(alice, chat)
client.wait_ready(bob, attachment)
oversized = client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=oversized.png",
202, actor=alice, raw=oversized_image(), headers={"Content-Type": "image/png"},
schema="Attachment")["json"]
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
oversized = client.call("GET", "/api/v1/attachments/" + oversized["id"], actor=alice, schema="Attachment")["json"]
if oversized["status"] in ("failed", "ready"):
break
time.sleep(0.5)
check(oversized["status"] == "failed", "oversized job did not fail with the default worker limit")
print("Worker: изображение 3000×3000 завершилось в состоянии failed.")
bad_auth = "Basic " + base64.b64encode(("unknown:" + marker).encode()).decode()
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": bad_auth})
forbidden = [
marker, alice["token"], alice["registration"]["password"], chat["id"],
message["json"]["id"], "sample.png", base64.b64encode(original).decode(),
]
def metrics(expr):
return source("prometheus", "/api/v1/query", query=expr)["data"]["result"]
def correlate(service, event, predicate, span_kind):
deadline = time.monotonic() + 45
while time.monotonic() < deadline:
logs = source("loki", "/loki/api/v1/query_range",
query='{service_name="' + service + '"} |= "' + event + '"',
start=start, end=time.time_ns(), limit=100)["data"]["result"]
for stream in logs:
for entry in stream["values"]:
metadata = {**stream["stream"], **(entry[2] if len(entry) > 2 else {})}
if not predicate(metadata):
continue
trace_id, span_id = metadata.get("trace_id", ""), metadata.get("span_id", "")
check(bool(re.fullmatch("[a-f0-9]{32}", trace_id)), "missing trace ID")
check(bool(re.fullmatch("[a-f0-9]{16}", span_id)), "missing span ID")
try:
trace = source("tempo", "/api/traces/" + trace_id)
except urllib.error.HTTPError as exc:
if exc.code == 404:
continue
raise
spans = [span for batch in trace["batches"] for scope in batch["scopeSpans"] for span in scope["spans"]]
span_id_base64 = base64.b64encode(bytes.fromhex(span_id)).decode()
matched = [span for span in spans if span["spanId"] in (span_id, span_id_base64)]
check(len(matched) == 1, "span ID from log missing in trace")
span = matched[0]
check(span["kind"] in (span_kind, "SPAN_KIND_SERVER" if span_kind == 2 else "SPAN_KIND_CONSUMER"), "wrong span kind")
check(int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"]), "span duration missing")
print(f"{service}: {event}, trace={trace_id}, span={span_id}")
return [logs, trace]
time.sleep(2)
raise Failure(service + ": correlated log and span not found")
# Loki may include each structured-metadata value in its own result stream.
api_data = correlate("messenger-api", "http.request.completed", lambda attrs: attrs.get("request_id") == request_id, 2)
worker_data = correlate("messenger-worker", "job.processing.completed", lambda attrs: attrs.get("job_outcome") == "success" and attrs.get("attachment_id") == attachment["id"], 5)
error_data = correlate("messenger-worker", "job.processing.completed",
lambda attrs: attrs.get("job_outcome") == "error" and attrs.get("attachment_id") == oversized["id"], 5)
expressions = [
"messenger_http_requests_total",
"messenger_http_duration_seconds_bucket",
'messenger_worker_jobs_total{job_outcome="success"}',
'messenger_worker_jobs_total{job_outcome="error"}',
"messenger_worker_duration_seconds_bucket",
]
series = []
for expression in expressions:
deadline = time.monotonic() + 30
samples = metrics(expression)
while not samples and time.monotonic() < deadline:
time.sleep(2)
samples = metrics(expression)
check(bool(samples), "missing metric: " + expression)
for sample in samples:
labels = sample["metric"]
check(not (labels.keys() & {"user_id", "chat_id", "message_id", "attachment_id", "request_id", "url", "filename", "text"}), "unbounded labels")
series.extend(samples)
print(f"Prometheus: {expression}, рядов={len(samples)}")
captured = json.dumps([api_data, worker_data, error_data, series], ensure_ascii=False)
check(all(value not in captured for value in forbidden), "sensitive test value in telemetry")
print("Проверенные сигналы не содержат контрольных credentials, текста, имени/байтов файла и бизнес-ID.")
if __name__ == "__main__":
try:
main()
except (Failure, OSError, ValueError, KeyError) as error:
print("FAIL:", error)
sys.exit(1)
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env python3
"""Сбор воспроизводимых доказательств уровня 3."""
import argparse
from pathlib import Path
import re
import shlex
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--grafana-url", default="http://127.0.0.1:3000")
args = parser.parse_args()
probe = [sys.executable, "scripts/check_observability.py", "--base-url", args.base_url,
"--grafana-url", args.grafana_url]
checks = [
("grade3-stand", "Стенд, тома и provisioning",
[["docker", "compose", "config", "--quiet"],
["docker", "compose", "ps", "--all"],
["docker", "compose", "config", "--volumes"],
["docker", "compose", "run", "--rm", "--no-deps", "otel-collector", "validate", "--config=/etc/otelcol-contrib/config.yaml"],
probe + ["provisioning"]],
"Prometheus хранит временные ряды, Loki — журналы, Tempo — трассы. Collector принимает и маршрутизирует OTLP. Grafana читает данные и загружает dashboard из JSON."),
("grade3-smoke", "Совместимость API при включённой телеметрии",
[["make", "check"], ["make", "test", "BASE_URL=" + args.base_url]],
"Smoke проверяет сессии, сообщения, ACL, идемпотентность и готовность вложения с оригиналом и thumbnail."),
("grade3-signals", "Метрики, структурные логи и трассы API и worker",
[probe + ["signals"]],
"Контрольный POST сопоставляется с логом по X-Request-Id. Оба сервиса сопоставляются с Tempo по trace_id и span_id. Prometheus возвращает счётчики и histogram buckets. Проверка ищет утечки синтетических значений в полученных сигналах."),
]
failed = False
for name, title, commands, explanation in checks:
lines = [f"# {title}", "", "Предпосылки — локальный синтетический стенд после make app-up и доступ к Docker.",
"", explanation, ""]
for command in commands:
try:
result = subprocess.run(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, timeout=180)
code, output = result.returncode, result.stdout
except subprocess.TimeoutExpired:
code, output = 124, "Истёк лимит времени."
except OSError:
code, output = 127, "Команда недоступна."
failed |= code != 0
output = re.sub(r"(https?://[^\s?]+)\?[^\s]+", r"\1?[redacted]", output).strip()
lines.extend([f"Команда `{shlex.join(command)}`. Код выхода **{code}**.", "", "```text", output or "(пусто)", "```", ""])
path = ROOT / "evidence" / (name + ".md")
path.write_text("\n".join(lines), encoding="utf-8")
print(path.relative_to(ROOT))
return int(failed)
if __name__ == "__main__":
sys.exit(main())
+8 -1
View File
@@ -17,6 +17,7 @@ import (
"messenger/src/backend/handlers"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"messenger/src/backend/telemetry"
)
func serverAddress() string {
@@ -88,9 +89,15 @@ func RunHTTP() error {
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
signals, stopTelemetry, err := telemetry.Start("messenger-api")
if err != nil {
return fmt.Errorf("telemetry initialization failed: %w", err)
}
defer stopTelemetry()
server := &http.Server{
Addr: serverAddress(),
Handler: mux,
Handler: signals.HTTP(mux),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
+116
View File
@@ -0,0 +1,116 @@
package telemetry
import (
"net/http"
"strconv"
"strings"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
type responseStatus struct {
http.ResponseWriter
status int
}
func (w *responseStatus) WriteHeader(status int) {
if w.status == 0 && status >= 200 {
w.status = status
}
w.ResponseWriter.WriteHeader(status)
}
func (w *responseStatus) Write(body []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
return w.ResponseWriter.Write(body)
}
func (w *responseStatus) Unwrap() http.ResponseWriter { return w.ResponseWriter }
func (s *Signals) HTTP(next http.Handler) http.Handler {
if s == nil {
return next
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
route := routeTemplate(r.URL.Path)
if route == "/health/live" || route == "/health/ready" {
next.ServeHTTP(w, r)
return
}
method := r.Method
switch method {
case "GET", "POST", "DELETE", "OPTIONS", "HEAD", "PUT", "PATCH":
default:
method = "_OTHER"
}
start := time.Now()
ctx, span := s.tracer.Start(r.Context(), method+" "+route, trace.WithSpanKind(trace.SpanKindServer))
defer span.End()
response := &responseStatus{ResponseWriter: w}
next.ServeHTTP(response, r.WithContext(ctx))
status := response.status
if status == 0 {
status = http.StatusOK
}
failed := status >= 500
attrs := []attribute.KeyValue{
attribute.String("http.request.method", method),
attribute.String("http.route", route),
attribute.String("http.status.class", strconv.Itoa(status/100)+"xx"),
}
span.SetAttributes(attrs...)
span.SetAttributes(attribute.Int("http.response.status_code", status))
if failed {
span.SetStatus(codes.Error, "")
}
s.requests.Add(ctx, 1, metric.WithAttributes(attrs...))
s.httpDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...))
s.event(ctx, "http.request.completed", failed,
otellog.String("http.request.method", method),
otellog.String("http.route", route),
otellog.Int("http.response.status_code", status),
otellog.String("request_id", w.Header().Get("X-Request-Id")))
})
}
func routeTemplate(path string) string {
switch path {
case "/health/live", "/health/ready", "/api/v1/users", "/api/v1/users/me",
"/api/v1/auth/sessions", "/api/v1/auth/sessions/current", "/api/v1/chats":
return path
}
parts := strings.Split(strings.TrimPrefix(path, "/"), "/")
if len(parts) < 4 || parts[0] != "api" || parts[1] != "v1" || parts[3] == "" {
return "unmatched"
}
switch parts[2] {
case "chats":
if len(parts) == 4 {
return "/api/v1/chats/{chat_id}"
}
if len(parts) == 5 && parts[4] == "attachment-uploads" {
return "/api/v1/chats/{chat_id}/attachment-uploads"
}
if len(parts) == 7 && parts[4] == "attachment-uploads" && parts[6] == "complete" {
return "/api/v1/chats/{chat_id}/attachment-uploads/{upload_id}/complete"
}
if len(parts) == 5 && (parts[4] == "messages" || parts[4] == "attachments") {
return "/api/v1/chats/{chat_id}/" + parts[4]
}
case "attachments":
if len(parts) == 4 {
return "/api/v1/attachments/{attachment_id}"
}
if len(parts) == 5 && parts[4] == "content" {
return "/api/v1/attachments/{attachment_id}/content"
}
}
return "unmatched"
}
+102
View File
@@ -0,0 +1,102 @@
package telemetry
import (
"context"
"errors"
"log"
"os"
"time"
"github.com/google/uuid"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
type Signals struct {
tracer trace.Tracer
logger otellog.Logger
requests metric.Int64Counter
httpDuration metric.Float64Histogram
jobs metric.Int64Counter
jobDuration metric.Float64Histogram
}
func Start(service string) (*Signals, func(), error) {
stop := func() {}
if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") == "" {
return nil, stop, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
instance, err := os.Hostname()
if err != nil || instance == "" {
instance = uuid.NewString()
}
res := resource.NewSchemaless(
attribute.String("service.name", service),
attribute.String("service.version", "lab6"),
attribute.String("service.instance.id", instance),
attribute.String("deployment.environment.name", "lab"),
)
traces, err := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(2*time.Second))
if err != nil {
return nil, stop, err
}
metrics, err := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(2*time.Second))
if err != nil {
_ = traces.Shutdown(ctx)
return nil, stop, err
}
logs, err := otlploghttp.New(ctx, otlploghttp.WithTimeout(2*time.Second))
if err != nil {
_ = traces.Shutdown(ctx)
_ = metrics.Shutdown(ctx)
return nil, stop, err
}
tp := sdktrace.NewTracerProvider(sdktrace.WithResource(res), sdktrace.WithBatcher(traces))
mp := sdkmetric.NewMeterProvider(sdkmetric.WithResource(res), sdkmetric.WithReader(
sdkmetric.NewPeriodicReader(metrics, sdkmetric.WithInterval(5*time.Second), sdkmetric.WithTimeout(3*time.Second))))
lp := sdklog.NewLoggerProvider(sdklog.WithResource(res), sdklog.WithProcessor(sdklog.NewBatchProcessor(logs)))
stop = func() {
deadline, cancel := context.WithTimeout(context.Background(), 4*time.Second)
defer cancel()
if errors.Join(tp.Shutdown(deadline), mp.Shutdown(deadline), lp.Shutdown(deadline)) != nil {
log.Print("telemetry shutdown did not flush all signals")
}
}
meter := mp.Meter("messenger")
buckets := metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10, 30, 60)
requests, errRequests := meter.Int64Counter("messenger.http.requests")
httpDuration, errHTTP := meter.Float64Histogram("messenger.http.duration", metric.WithUnit("s"), buckets)
jobs, errJobs := meter.Int64Counter("messenger.worker.jobs")
jobDuration, errJobDuration := meter.Float64Histogram("messenger.worker.duration", metric.WithUnit("s"), buckets)
if err := errors.Join(errRequests, errHTTP, errJobs, errJobDuration); err != nil {
stop()
return nil, func() {}, err
}
return &Signals{tp.Tracer("messenger"), lp.Logger("messenger"), requests, httpDuration, jobs, jobDuration}, stop, nil
}
func (s *Signals) event(ctx context.Context, event string, failed bool, attrs ...otellog.KeyValue) {
var record otellog.Record
record.SetTimestamp(time.Now())
record.SetSeverity(otellog.SeverityInfo)
record.SetSeverityText("INFO")
if failed {
record.SetSeverity(otellog.SeverityError)
record.SetSeverityText("ERROR")
}
record.AddAttributes(otellog.String("event", event))
record.SetBody(otellog.StringValue(event))
record.AddAttributes(attrs...)
s.logger.Emit(ctx, record)
}
+32
View File
@@ -0,0 +1,32 @@
package telemetry
import (
"context"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
otellog "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
)
func (s *Signals) Job(ctx context.Context, jobType, attachmentID string) (context.Context, func(string)) {
if s == nil {
return ctx, func(string) {}
}
start := time.Now()
ctx, span := s.tracer.Start(ctx, jobType+" process", trace.WithSpanKind(trace.SpanKindConsumer))
return ctx, func(outcome string) {
defer span.End()
attrs := []attribute.KeyValue{attribute.String("job.type", jobType), attribute.String("job.outcome", outcome)}
span.SetAttributes(attrs...)
failed := outcome == "error"
if failed {
span.SetStatus(codes.Error, "")
}
s.jobs.Add(ctx, 1, metric.WithAttributes(attrs...))
s.jobDuration.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...))
s.event(ctx, "job.processing.completed", failed, otellog.String("job.type", jobType), otellog.String("job.outcome", outcome), otellog.String("attachment_id", attachmentID))
}
}
+25 -3
View File
@@ -14,6 +14,7 @@ import (
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"messenger/src/backend/telemetry"
)
func RunPipeline(mode string, args []string) error {
@@ -58,13 +59,22 @@ func RunPipeline(mode string, args []string) error {
if err != nil {
return err
}
var signals *telemetry.Signals
if mode == "worker" {
var stopTelemetry func()
signals, stopTelemetry, err = telemetry.Start("messenger-worker")
if err != nil {
return err
}
defer stopTelemetry()
}
for ctx.Err() == nil {
broker, e := repositories.NewJobBroker(ctx, address)
if e == nil {
if mode == "dispatcher" {
e = dispatchJobs(ctx, store, broker)
} else {
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout)
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout, signals)
}
broker.Close()
}
@@ -110,7 +120,7 @@ func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, brok
return nil
}
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration) error {
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration, signals *telemetry.Signals) error {
deliveries, err := broker.Consume()
if err != nil {
return errors.New("RabbitMQ consumer unavailable")
@@ -132,12 +142,16 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
continue
}
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
task, finish := signals.Job(task, job.Type, job.AttachmentID)
var imageErr error
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
if workErr == nil && token != "" {
faults.Crash("before_effect", job.AttachmentID)
faults.Pause(task, "after_claim", job.AttachmentID)
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return processImage(task, objects, a, staging)
variants, code, err := processImage(task, objects, a, staging)
imageErr = err
return variants, code, err
})
}
cancel()
@@ -148,6 +162,14 @@ func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objec
if workErr == nil && token == "" || errors.Is(workErr, repositories.ErrStaleJob) {
completed, workErr = store.JobTerminal(ctx, job.ID)
}
outcome := "success"
switch {
case workErr != nil || imageErr != nil:
outcome = "error"
case token == "" || !completed:
outcome = "skipped"
}
finish(outcome)
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
// A closed connection automatically returns unacked deliveries to RabbitMQ.
if completed {
+27 -7
View File
@@ -11,18 +11,33 @@ require (
github.com/minio/minio-go/v7 v7.0.76
github.com/rabbitmq/amqp091-go v1.10.0
github.com/redis/go-redis/v9 v9.17.3
golang.org/x/crypto v0.27.0
go.opentelemetry.io/otel v1.34.0
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.10.0
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0
go.opentelemetry.io/otel/log v0.10.0
go.opentelemetry.io/otel/metric v1.34.0
go.opentelemetry.io/otel/sdk v1.34.0
go.opentelemetry.io/otel/sdk/log v0.10.0
go.opentelemetry.io/otel/sdk/metric v1.34.0
go.opentelemetry.io/otel/trace v1.34.0
golang.org/x/crypto v0.32.0
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8
)
require (
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ini/ini v1.67.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/goccy/go-json v0.10.3 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa // indirect
@@ -51,11 +66,16 @@ require (
github.com/tklauser/go-sysconf v0.3.14 // indirect
github.com/tklauser/numcpus v0.8.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opentelemetry.io/auto/sdk v1.1.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect
golang.org/x/net v0.29.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/text v0.18.0 // indirect
google.golang.org/protobuf v1.34.2 // indirect
golang.org/x/net v0.34.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.29.0 // indirect
golang.org/x/text v0.21.0 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
google.golang.org/grpc v1.69.4 // indirect
google.golang.org/protobuf v1.36.3 // indirect
)
+51 -20
View File
@@ -6,6 +6,8 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -31,6 +33,7 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
@@ -53,6 +56,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 h1:VNqngBF40hVlDloBruUehVYC3ArSgIyScOAyMRqBxRg=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1/go.mod h1:RBRO7fro65R6tjKzYgLAFo0t1QEXY1Dp+i/bvpRiqiQ=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -135,8 +140,8 @@ github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnj
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tinylib/msgp v1.2.1 h1:6ypy2qcCznxpP4hpORzhtXyTqrBs7cfM9MCCWY8zsmU=
github.com/tinylib/msgp v1.2.1/go.mod h1:2vIGs3lcUo8izAATNobrCHevYZC/LMsJtw4JPiYPHro=
github.com/tklauser/go-sysconf v0.3.14 h1:g5vzr9iPFFz24v2KZXs/pvpvh8/V9Fw6vQK5ZZb78yU=
@@ -145,32 +150,52 @@ github.com/tklauser/numcpus v0.8.0 h1:Mx4Wwe/FjZLeQsK/6kt2EOepwwSl7SmJrK5bV/dXYg
github.com/tklauser/numcpus v0.8.0/go.mod h1:ZJZlAY+dmR4eut8epnzf0u/VwodKmryxR8txiloSqBE=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4=
go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q=
go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s=
go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g=
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.10.0 h1:q/heq5Zh8xV1+7GoMGJpTxM2Lhq5+bFxB29tshuRuw0=
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.10.0/go.mod h1:leO2CSTg0Y+LyvmR7Wm4pUxE8KAmaM2GCVx7O+RATLA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0 h1:opwv08VbCZ8iecIWs+McMdHRcAXzjAeda3uG2kI/hcA=
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.34.0/go.mod h1:oOP3ABpW7vFHulLpE8aYtNBodrHhMTrvfxUXGvqm7Ac=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 h1:OeNbIYk/2C15ckl7glBlOBp5+WlYsOElzTNmiPW/x60=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0/go.mod h1:7Bept48yIeqxP2OZ9/AqIpYS94h2or0aB4FypJTc8ZM=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0 h1:BEj3SPM81McUZHYjRS5pEgNgnmzGJ5tRpU5krWnV8Bs=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.34.0/go.mod h1:9cKLGBDzI/F3NoHLQGm4ZrYdIHsvGt6ej6hUowxY0J4=
go.opentelemetry.io/otel/log v0.10.0 h1:1CXmspaRITvFcjA4kyVszuG4HjA61fPDxMb7q3BuyF0=
go.opentelemetry.io/otel/log v0.10.0/go.mod h1:PbVdm9bXKku/gL0oFfUF4wwsQsOPlpo4VEqjvxih+FM=
go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ=
go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE=
go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A=
go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU=
go.opentelemetry.io/otel/sdk/log v0.10.0 h1:lR4teQGWfeDVGoute6l0Ou+RpFqQ9vaPdrNJlST0bvw=
go.opentelemetry.io/otel/sdk/log v0.10.0/go.mod h1:A+V1UTWREhWAittaQEG4bYm4gAZa6xnvVu+xKrIRkzo=
go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk=
go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w=
go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k=
go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE=
go.opentelemetry.io/proto/otlp v1.5.0 h1:xJvq7gMzB31/d406fB8U5CBdyQGw4P399D1aQWU/3i4=
go.opentelemetry.io/proto/otlp v1.5.0/go.mod h1:keN8WnHxOy8PG0rQZjJJ5A2ebUoafqWp0eVQ4yIXvJ4=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U=
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo=
golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0=
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -179,15 +204,21 @@ golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f h1:gap6+3Gk41EItBuyi4XX/bp4oqJ3UwuIMl25yGinuAA=
google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:Ic02D47M+zbarjYYUlK57y316f2MoN0gjAwI3f2S95o=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI=
google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50=
google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A=
google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU=
google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=