ops: add observability stack and SLO alerts

This commit is contained in:
2026-11-01 17:06:35 +03:00
parent 934dfefee7
commit 4bf383bd51
15 changed files with 1341 additions and 32 deletions
+94 -9
View File
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
target: /etc/messenger/messenger.yaml
read_only: true
x-telemetry-environment: &telemetry-environment
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
services:
postgres:
image: postgres:17.6-alpine3.22
@@ -118,7 +123,7 @@ services:
networks: [data]
migrate:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "migrate"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -137,8 +142,10 @@ services:
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-api
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
@@ -176,9 +183,11 @@ services:
networks: [application, data]
dispatcher:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "dispatch"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-dispatcher
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
BROKER_URL: ${BROKER_URL}
@@ -202,9 +211,11 @@ services:
networks: [data, monitoring]
worker:
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/messenger", "worker"]
environment:
<<: *telemetry-environment
OTEL_SERVICE_NAME: messenger-worker
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
DATABASE_URL: ${DATABASE_URL}
S3_ENDPOINT: ${S3_ENDPOINT}
@@ -239,7 +250,7 @@ services:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
@@ -259,7 +270,7 @@ services:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
@@ -289,7 +300,7 @@ services:
lab5-experiment:
profiles: ["experiments"]
image: ${API_IMAGE:-mtusi-messenger:lab5}
image: ${API_IMAGE:-mtusi-messenger:lab6}
command: ["/app/lab5experiment"]
environment:
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
@@ -375,11 +386,12 @@ services:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
- --web.enable-remote-write-receiver
volumes:
- prometheus_data:/prometheus
ports:
@@ -398,6 +410,9 @@ services:
- dispatcher
- worker
- s3
- otel-collector
- loki
- tempo
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
@@ -407,28 +422,98 @@ services:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
GF_EXPLORE_ENABLED: "true"
volumes:
- grafana_data:/var/lib/grafana
ports:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
- loki
- tempo
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
loki:
image: grafana/loki:3.7.9
command: ["-config.file=/etc/loki/loki.yaml"]
volumes:
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
- loki_data:/loki
ports:
- "${LOKI_BIND_ADDRESS}:3100"
healthcheck:
test: ["CMD", "/usr/bin/loki", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${LOKI_CPUS}
mem_limit: ${LOKI_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
tempo:
image: grafana/tempo:2.10.10
command: ["-config.file=/etc/tempo/tempo.yaml"]
volumes:
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
- tempo_data:/var/tempo
ports:
- "${TEMPO_BIND_ADDRESS}:3200"
healthcheck:
test: ["CMD", "/tempo", "-version"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${TEMPO_CPUS}
mem_limit: ${TEMPO_MEMORY}
restart: unless-stopped
networks: [edge, monitoring]
otel-collector:
image: otel/opentelemetry-collector-contrib:0.160.0
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
volumes:
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
expose:
- "4317"
- "4318"
- "8888"
- "8889"
- "13133"
depends_on:
- loki
- tempo
healthcheck:
test: ["CMD", "/otelcol-contrib", "components"]
interval: 10s
timeout: 3s
retries: 10
cpus: ${OTEL_COLLECTOR_CPUS}
mem_limit: ${OTEL_COLLECTOR_MEMORY}
restart: unless-stopped
networks: [application, data, monitoring]
volumes:
postgres_data:
valkey_data:
redis_broker_data:
prometheus_data:
grafana_data:
loki_data:
tempo_data:
s3_data:
networks: