feat: add otel collector, prometheus, loki, tempo and grafana

This commit is contained in:
2026-11-18 19:23:20 +03:00
parent 98d5787545
commit 9ed32e2d15
7 changed files with 343 additions and 3 deletions
+106 -3
View File
@@ -10,8 +10,8 @@ services:
- -ec
- |
cd /secrets
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24; do
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker grafana
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24 admin_password:grafana:16; do
file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:}
[ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
done
@@ -28,7 +28,7 @@ services:
printf 'default_user = messenger\ndefault_pass = %s\nloopback_users = none\nlog.console = true\nlisteners.tcp.default = 5672\n' "$$(cat broker/broker_password)" > rabbitmq/10-messenger.conf
cp api/database_url api/s3_access_key_id api/s3_secret_access_key reconciler/
cp api/s3_access_key_id api/s3_secret_access_key worker/
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/*
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* grafana/*
deploy:
resources:
limits: { cpus: "0.1", memory: 32M }
@@ -310,6 +310,101 @@ services:
condition: service_healthy
restart: unless-stopped
otel-collector:
image: otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8
command: ["--config=/etc/otelcol/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
networks: [default, telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
restart: unless-stopped
prometheus:
image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --storage.tsdb.retention.size=1GB
- --web.enable-otlp-receiver
volumes:
- ./infra/prometheus:/etc/prometheus:ro
- prometheusdata:/prometheus
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
loki:
image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- lokidata:/loki
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
restart: unless-stopped
tempo:
image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempodata:/var/tempo
networks: [telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 512M }
restart: unless-stopped
grafana:
image: grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
ports:
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/admin_password
GF_AUTH_ANONYMOUS_ENABLED: "true"
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
GF_USERS_VIEWERS_CAN_EDIT: "true"
GF_ANALYTICS_REPORTING_ENABLED: "false"
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
GF_NEWS_NEWS_FEED_ENABLED: "false"
volumes:
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafanadata:/var/lib/grafana
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: grafana } }
networks: [default, telemetry]
deploy:
resources:
limits: { cpus: "0.5", memory: 256M }
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/health"]
interval: 5s
timeout: 3s
retries: 20
depends_on:
secrets-init:
condition: service_completed_successfully
prometheus:
condition: service_healthy
loki:
condition: service_started
tempo:
condition: service_started
restart: unless-stopped
tools:
profiles: [tools]
build:
@@ -331,4 +426,12 @@ volumes:
pgdata:
s3data:
rabbitmqdata:
prometheusdata:
lokidata:
tempodata:
grafanadata:
secrets:
networks:
telemetry:
internal: true
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: messenger
folder: Messenger
type: file
disableDeletion: true
allowUiUpdates: false
options:
path: /var/lib/grafana/dashboards
@@ -0,0 +1,45 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
jsonData:
httpMethod: POST
exemplarTraceIdDestinations:
- name: trace_id
datasourceUid: tempo
- name: Loki
uid: loki
type: loki
access: proxy
url: http://loki:3100
jsonData:
derivedFields:
- name: trace_id
matcherType: label
matcherRegex: trace_id
url: "$${__value.raw}"
datasourceUid: tempo
- name: Tempo
uid: tempo
type: tempo
access: proxy
url: http://tempo:3200
jsonData:
tracesToLogsV2:
datasourceUid: loki
spanStartTimeShift: -5m
spanEndTimeShift: 5m
filterByTraceID: false
customQuery: true
query: '{service_name=~".+"} | trace_id="$${__span.traceId}"'
tracesToMetrics:
datasourceUid: prometheus
serviceMap:
datasourceUid: prometheus
nodeGraph:
enabled: true
+43
View File
@@ -0,0 +1,43 @@
auth_enabled: false
server:
http_listen_address: 0.0.0.0
http_listen_port: 3100
grpc_listen_port: 9096
log_level: warn
common:
instance_addr: 127.0.0.1
path_prefix: /loki
replication_factor: 1
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2026-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true
retention_period: 168h
ingestion_rate_mb: 8
ingestion_burst_size_mb: 16
compactor:
working_directory: /loki/compactor
retention_enabled: true
delete_request_store: filesystem
analytics:
reporting_enabled: false
+85
View File
@@ -0,0 +1,85 @@
extensions:
health_check:
endpoint: 0.0.0.0:13133
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
max_request_body_size: 4194304
processors:
memory_limiter:
check_interval: 1s
limit_mib: 160
spike_limit_mib: 40
batch:
send_batch_size: 512
send_batch_max_size: 1024
timeout: 2s
exporters:
otlphttp/prometheus:
endpoint: http://prometheus:9090/api/v1/otlp
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/loki:
endpoint: http://loki:3100/otlp
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
otlphttp/tempo:
endpoint: http://tempo:4318
timeout: 5s
sending_queue:
enabled: true
num_consumers: 2
queue_size: 200
retry_on_failure:
enabled: true
initial_interval: 1s
max_interval: 10s
max_elapsed_time: 60s
service:
extensions: [health_check]
telemetry:
logs:
level: warn
metrics:
level: normal
readers:
- pull:
exporter:
prometheus:
host: 0.0.0.0
port: 8888
pipelines:
metrics:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/prometheus]
logs:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/loki]
traces:
receivers: [otlp]
processors: [memory_limiter, batch]
exporters: [otlphttp/tempo]
+23
View File
@@ -0,0 +1,23 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
otlp:
promote_resource_attributes:
- deployment.environment.name
- service.version
storage:
tsdb:
out_of_order_time_window: 30m
rule_files:
- /etc/prometheus/rules/*.yml
scrape_configs:
- job_name: otel-collector
static_configs:
- targets: ["otel-collector:8888"]
- job_name: prometheus
static_configs:
- targets: ["localhost:9090"]
+31
View File
@@ -0,0 +1,31 @@
stream_over_http_enabled: true
server:
http_listen_address: 0.0.0.0
http_listen_port: 3200
log_level: warn
distributor:
receivers:
otlp:
protocols:
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_duration: 5m
compactor:
compaction:
block_retention: 168h
storage:
trace:
backend: local
wal:
path: /var/tempo/wal
local:
path: /var/tempo/blocks
usage_report:
reporting_enabled: false