forked from mtusi2026/mtusi2026lab6
feat: add otel collector, prometheus, loki, tempo and grafana
This commit is contained in:
+106
-3
@@ -10,8 +10,8 @@ services:
|
||||
- -ec
|
||||
- |
|
||||
cd /secrets
|
||||
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker
|
||||
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24; do
|
||||
mkdir -p postgres redis api migrate s3 s3-init reconciler broker rabbitmq relay worker grafana
|
||||
for entry in pg_password:postgres:24 redis_password:redis:24 s3_root_user:s3:10 s3_root_password:s3:20 s3_access_key_id:api:10 s3_secret_access_key:api:20 broker_password:broker:24 admin_password:grafana:16; do
|
||||
file=$${entry%%:*}; rest=$${entry#*:}; dir=$${rest%%:*}; size=$${rest##*:}
|
||||
[ -s "$$dir/$$file" ] || head -c "$$size" /dev/urandom | od -An -tx1 | tr -d ' \n' > "$$dir/$$file"
|
||||
done
|
||||
@@ -28,7 +28,7 @@ services:
|
||||
printf 'default_user = messenger\ndefault_pass = %s\nloopback_users = none\nlog.console = true\nlisteners.tcp.default = 5672\n' "$$(cat broker/broker_password)" > rabbitmq/10-messenger.conf
|
||||
cp api/database_url api/s3_access_key_id api/s3_secret_access_key reconciler/
|
||||
cp api/s3_access_key_id api/s3_secret_access_key worker/
|
||||
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/*
|
||||
chmod 0444 postgres/* redis/* api/* migrate/* s3/* s3-init/* reconciler/* broker/* rabbitmq/* relay/* worker/* grafana/*
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.1", memory: 32M }
|
||||
@@ -310,6 +310,101 @@ services:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.115.1@sha256:d2da12c4336a79758826700be9e21ecf4a9f7d945b7f8a58ba55ee3fa45427c8
|
||||
command: ["--config=/etc/otelcol/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
||||
networks: [default, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
restart: unless-stopped
|
||||
|
||||
prometheus:
|
||||
image: prom/prometheus:v3.1.0@sha256:6559acbd5d770b15bb3c954629ce190ac3cbbdb2b7f1c30f0385c4e05104e218
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --storage.tsdb.retention.size=1GB
|
||||
- --web.enable-otlp-receiver
|
||||
volumes:
|
||||
- ./infra/prometheus:/etc/prometheus:ro
|
||||
- prometheusdata:/prometheus
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.3.2@sha256:8af2de1abbdd7aa92b27c9bcc96f0f4140c9096b507c77921ffddf1c6ad6c48f
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- lokidata:/loki
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
restart: unless-stopped
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.6.1@sha256:ef4384fce6e8ad22b95b243d8fc165628cda655376fd50e7850536ad89d71d50
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempodata:/var/tempo
|
||||
networks: [telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
restart: unless-stopped
|
||||
|
||||
grafana:
|
||||
image: grafana/grafana:11.4.0@sha256:d8ea37798ccc41061a62ab080f2676dda6bf7815558499f901bdb0f533a456fb
|
||||
ports:
|
||||
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000"
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/admin_password
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
|
||||
GF_USERS_VIEWERS_CAN_EDIT: "true"
|
||||
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
||||
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
||||
GF_NEWS_NEWS_FEED_ENABLED: "false"
|
||||
volumes:
|
||||
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
||||
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
||||
- grafanadata:/var/lib/grafana
|
||||
- { type: volume, source: secrets, target: /run/secrets, read_only: true, volume: { subpath: grafana } }
|
||||
networks: [default, telemetry]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:3000/api/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
depends_on:
|
||||
secrets-init:
|
||||
condition: service_completed_successfully
|
||||
prometheus:
|
||||
condition: service_healthy
|
||||
loki:
|
||||
condition: service_started
|
||||
tempo:
|
||||
condition: service_started
|
||||
restart: unless-stopped
|
||||
|
||||
tools:
|
||||
profiles: [tools]
|
||||
build:
|
||||
@@ -331,4 +426,12 @@ volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
rabbitmqdata:
|
||||
prometheusdata:
|
||||
lokidata:
|
||||
tempodata:
|
||||
grafanadata:
|
||||
secrets:
|
||||
|
||||
networks:
|
||||
telemetry:
|
||||
internal: true
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
apiVersion: 1
|
||||
|
||||
providers:
|
||||
- name: messenger
|
||||
folder: Messenger
|
||||
type: file
|
||||
disableDeletion: true
|
||||
allowUiUpdates: false
|
||||
options:
|
||||
path: /var/lib/grafana/dashboards
|
||||
@@ -0,0 +1,45 @@
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
jsonData:
|
||||
httpMethod: POST
|
||||
exemplarTraceIdDestinations:
|
||||
- name: trace_id
|
||||
datasourceUid: tempo
|
||||
- name: Loki
|
||||
uid: loki
|
||||
type: loki
|
||||
access: proxy
|
||||
url: http://loki:3100
|
||||
jsonData:
|
||||
derivedFields:
|
||||
- name: trace_id
|
||||
matcherType: label
|
||||
matcherRegex: trace_id
|
||||
url: "$${__value.raw}"
|
||||
datasourceUid: tempo
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
jsonData:
|
||||
tracesToLogsV2:
|
||||
datasourceUid: loki
|
||||
spanStartTimeShift: -5m
|
||||
spanEndTimeShift: 5m
|
||||
filterByTraceID: false
|
||||
customQuery: true
|
||||
query: '{service_name=~".+"} | trace_id="$${__span.traceId}"'
|
||||
tracesToMetrics:
|
||||
datasourceUid: prometheus
|
||||
serviceMap:
|
||||
datasourceUid: prometheus
|
||||
nodeGraph:
|
||||
enabled: true
|
||||
@@ -0,0 +1,43 @@
|
||||
auth_enabled: false
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3100
|
||||
grpc_listen_port: 9096
|
||||
log_level: warn
|
||||
|
||||
common:
|
||||
instance_addr: 127.0.0.1
|
||||
path_prefix: /loki
|
||||
replication_factor: 1
|
||||
storage:
|
||||
filesystem:
|
||||
chunks_directory: /loki/chunks
|
||||
rules_directory: /loki/rules
|
||||
ring:
|
||||
kvstore:
|
||||
store: inmemory
|
||||
|
||||
schema_config:
|
||||
configs:
|
||||
- from: 2026-01-01
|
||||
store: tsdb
|
||||
object_store: filesystem
|
||||
schema: v13
|
||||
index:
|
||||
prefix: index_
|
||||
period: 24h
|
||||
|
||||
limits_config:
|
||||
allow_structured_metadata: true
|
||||
retention_period: 168h
|
||||
ingestion_rate_mb: 8
|
||||
ingestion_burst_size_mb: 16
|
||||
|
||||
compactor:
|
||||
working_directory: /loki/compactor
|
||||
retention_enabled: true
|
||||
delete_request_store: filesystem
|
||||
|
||||
analytics:
|
||||
reporting_enabled: false
|
||||
@@ -0,0 +1,85 @@
|
||||
extensions:
|
||||
health_check:
|
||||
endpoint: 0.0.0.0:13133
|
||||
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
max_request_body_size: 4194304
|
||||
|
||||
processors:
|
||||
memory_limiter:
|
||||
check_interval: 1s
|
||||
limit_mib: 160
|
||||
spike_limit_mib: 40
|
||||
batch:
|
||||
send_batch_size: 512
|
||||
send_batch_max_size: 1024
|
||||
timeout: 2s
|
||||
|
||||
exporters:
|
||||
otlphttp/prometheus:
|
||||
endpoint: http://prometheus:9090/api/v1/otlp
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/loki:
|
||||
endpoint: http://loki:3100/otlp
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
otlphttp/tempo:
|
||||
endpoint: http://tempo:4318
|
||||
timeout: 5s
|
||||
sending_queue:
|
||||
enabled: true
|
||||
num_consumers: 2
|
||||
queue_size: 200
|
||||
retry_on_failure:
|
||||
enabled: true
|
||||
initial_interval: 1s
|
||||
max_interval: 10s
|
||||
max_elapsed_time: 60s
|
||||
|
||||
service:
|
||||
extensions: [health_check]
|
||||
telemetry:
|
||||
logs:
|
||||
level: warn
|
||||
metrics:
|
||||
level: normal
|
||||
readers:
|
||||
- pull:
|
||||
exporter:
|
||||
prometheus:
|
||||
host: 0.0.0.0
|
||||
port: 8888
|
||||
pipelines:
|
||||
metrics:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/prometheus]
|
||||
logs:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/loki]
|
||||
traces:
|
||||
receivers: [otlp]
|
||||
processors: [memory_limiter, batch]
|
||||
exporters: [otlphttp/tempo]
|
||||
@@ -0,0 +1,23 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
otlp:
|
||||
promote_resource_attributes:
|
||||
- deployment.environment.name
|
||||
- service.version
|
||||
|
||||
storage:
|
||||
tsdb:
|
||||
out_of_order_time_window: 30m
|
||||
|
||||
rule_files:
|
||||
- /etc/prometheus/rules/*.yml
|
||||
|
||||
scrape_configs:
|
||||
- job_name: otel-collector
|
||||
static_configs:
|
||||
- targets: ["otel-collector:8888"]
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ["localhost:9090"]
|
||||
@@ -0,0 +1,31 @@
|
||||
stream_over_http_enabled: true
|
||||
|
||||
server:
|
||||
http_listen_address: 0.0.0.0
|
||||
http_listen_port: 3200
|
||||
log_level: warn
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
compactor:
|
||||
compaction:
|
||||
block_retention: 168h
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
|
||||
usage_report:
|
||||
reporting_enabled: false
|
||||
Reference in New Issue
Block a user