From 9ed32e2d15fcbfec571ecab801910ae1c1b9b175 Mon Sep 17 00:00:00 2001 From: aegor Date: Wed, 18 Nov 2026 19:23:20 +0300 Subject: [PATCH] feat: add otel collector, prometheus, loki, tempo and grafana --- compose.yaml | 109 +++++++++++++++++- .../provisioning/dashboards/dashboards.yaml | 10 ++ .../provisioning/datasources/datasources.yaml | 45 ++++++++ infra/loki/loki.yaml | 43 +++++++ infra/otel/collector.yaml | 85 ++++++++++++++ infra/prometheus/prometheus.yml | 23 ++++ infra/tempo/tempo.yaml | 31 +++++ 7 files changed, 343 insertions(+), 3 deletions(-) create mode 100644 infra/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/loki/loki.yaml create mode 100644 infra/otel/collector.yaml create mode 100644 infra/prometheus/prometheus.yml create mode 100644 infra/tempo/tempo.yaml diff --git a/compose.yaml b/compose.yaml index dcce9f8..7e95960 100644 --- a/compose.yaml +++ b/compose.yaml @@ -10,8 +10,8 @@ services: - -ec - | cd /secrets - mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker - for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24; do + mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker grafana + for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24 admin_password:grafana:16; do file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:} [ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file" done @@ -28,7 +28,7 @@ services: printf 'default_user = messenger\ndefault_pass = %s\nloopback_users = none\nlog.console = true\nlisteners.tcp.default = 5672\n' "$$(cat broker/broker_password)" > rabbitmq/10-messenger.conf cp api/database_url api/s3_access_key_id api/s3_secret_access_key reconciler/ cp api/s3_access_key_id api/s3_secret_access_key worker/ - chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* + chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* grafana/* deploy: resources: limits: { cpus: "0.1", memory: 32M } @@ -310,6 +310,101 @@ services: condition: service_healthy restart: unless-stopped + otel-collector: + image: otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8 + command: ["--config=/etc/otelcol/collector.yaml"] + volumes: + - ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro + networks: [default, telemetry] + deploy: + resources: + limits: { cpus: "0.5", memory: 256M } + restart: unless-stopped + + prometheus: + image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218 + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + - --storage.tsdb.retention.size=1GB + - --web.enable-otlp-receiver + volumes: + - ./infra/prometheus:/etc/prometheus:ro + - prometheusdata:/prometheus + networks: [telemetry] + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + loki: + image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro + - lokidata:/loki + networks: [telemetry] + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + restart: unless-stopped + + tempo: + image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50 + command: ["-config.file=/etc/tempo/tempo.yaml"] + volumes: + - ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro + - tempodata:/var/tempo + networks: [telemetry] + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + restart: unless-stopped + + grafana: + image: grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/admin_password + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_USERS_VIEWERS_CAN_EDIT: "true" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_NEWS_NEWS_FEED_ENABLED: "false" + volumes: + - ./infra/grafana/provisioning:/etc/grafana/provisioning:ro + - ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafanadata:/var/lib/grafana + - { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: grafana } } + networks: [default, telemetry] + deploy: + resources: + limits: { cpus: "0.5", memory: 256M } + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 20 + depends_on: + secrets-init: + condition: service_completed_successfully + prometheus: + condition: service_healthy + loki: + condition: service_started + tempo: + condition: service_started + restart: unless-stopped + tools: profiles: [tools] build: @@ -331,4 +426,12 @@ volumes: pgdata: s3data: rabbitmqdata: + prometheusdata: + lokidata: + tempodata: + grafanadata: secrets: + +networks: + telemetry: + internal: true diff --git a/infra/grafana/provisioning/dashboards/dashboards.yaml b/infra/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..a9d058c --- /dev/null +++ b/infra/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: messenger + folder: Messenger + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /var/lib/grafana/dashboards diff --git a/infra/grafana/provisioning/datasources/datasources.yaml b/infra/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..1abcca9 --- /dev/null +++ b/infra/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,45 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + jsonData: + httpMethod: POST + exemplarTraceIdDestinations: + - name: trace_id + datasourceUid: tempo + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + derivedFields: + - name: trace_id + matcherType: label + matcherRegex: trace_id + url: "$${__value.raw}" + datasourceUid: tempo + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 + jsonData: + tracesToLogsV2: + datasourceUid: loki + spanStartTimeShift: -5m + spanEndTimeShift: 5m + filterByTraceID: false + customQuery: true + query: '{service_name=~".+"} | trace_id="$${__span.traceId}"' + tracesToMetrics: + datasourceUid: prometheus + serviceMap: + datasourceUid: prometheus + nodeGraph: + enabled: true diff --git a/infra/loki/loki.yaml b/infra/loki/loki.yaml new file mode 100644 index 0000000..fe0639f --- /dev/null +++ b/infra/loki/loki.yaml @@ -0,0 +1,43 @@ +auth_enabled: false + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: warn + +common: + instance_addr: 127.0.0.1 + path_prefix: /loki + replication_factor: 1 + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: 2026-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + allow_structured_metadata: true + retention_period: 168h + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml new file mode 100644 index 0000000..30c8ccd --- /dev/null +++ b/infra/otel/collector.yaml @@ -0,0 +1,85 @@ +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + max_request_body_size: 4194304 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 160 + spike_limit_mib: 40 + batch: + send_batch_size: 512 + send_batch_max_size: 1024 + timeout: 2s + +exporters: + otlphttp/prometheus: + endpoint: http://prometheus:9090/api/v1/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlphttp/loki: + endpoint: http://loki:3100/otlp + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + otlphttp/tempo: + endpoint: http://tempo:4318 + timeout: 5s + sending_queue: + enabled: true + num_consumers: 2 + queue_size: 200 + retry_on_failure: + enabled: true + initial_interval: 1s + max_interval: 10s + max_elapsed_time: 60s + +service: + extensions: [health_check] + telemetry: + logs: + level: warn + metrics: + level: normal + readers: + - pull: + exporter: + prometheus: + host: 0.0.0.0 + port: 8888 + pipelines: + metrics: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/prometheus] + logs: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/loki] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/tempo] diff --git a/infra/prometheus/prometheus.yml b/infra/prometheus/prometheus.yml new file mode 100644 index 0000000..28f48eb --- /dev/null +++ b/infra/prometheus/prometheus.yml @@ -0,0 +1,23 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +otlp: + promote_resource_attributes: + - deployment.environment.name + - service.version + +storage: + tsdb: + out_of_order_time_window: 30m + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ["otel-collector:8888"] + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] diff --git a/infra/tempo/tempo.yaml b/infra/tempo/tempo.yaml new file mode 100644 index 0000000..4ae59a9 --- /dev/null +++ b/infra/tempo/tempo.yaml @@ -0,0 +1,31 @@ +stream_over_http_enabled: true + +server: + http_listen_address: 0.0.0.0 + http_listen_port: 3200 + log_level: warn + +distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +compactor: + compaction: + block_retention: 168h + +storage: + trace: + backend: local + wal: + path: /var/tempo/wal + local: + path: /var/tempo/blocks + +usage_report: + reporting_enabled: false