ops: add observability stack and SLO alerts
This commit is contained in:
+94
-9
@@ -4,6 +4,11 @@ x-messenger-config-volume: &messenger-config-volume
|
||||
target: /etc/messenger/messenger.yaml
|
||||
read_only: true
|
||||
|
||||
x-telemetry-environment: &telemetry-environment
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT}
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: ${OTEL_EXPORTER_OTLP_PROTOCOL}
|
||||
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=${DEPLOYMENT_ENVIRONMENT},service.version=${SERVICE_VERSION}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.6-alpine3.22
|
||||
@@ -118,7 +123,7 @@ services:
|
||||
networks: [data]
|
||||
|
||||
migrate:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "migrate"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -137,8 +142,10 @@ services:
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-api
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
APP_HOST: ${APP_HOST}
|
||||
APP_PORT: ${APP_PORT}
|
||||
@@ -176,9 +183,11 @@ services:
|
||||
networks: [application, data]
|
||||
|
||||
dispatcher:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "dispatch"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-dispatcher
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
BROKER_URL: ${BROKER_URL}
|
||||
@@ -202,9 +211,11 @@ services:
|
||||
networks: [data, monitoring]
|
||||
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/messenger", "worker"]
|
||||
environment:
|
||||
<<: *telemetry-environment
|
||||
OTEL_SERVICE_NAME: messenger-worker
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
DATABASE_URL: ${DATABASE_URL}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT}
|
||||
@@ -239,7 +250,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: loadbalancer
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab5}
|
||||
image: ${NGINX_IMAGE:-mtusi-nginx:lab6}
|
||||
ports:
|
||||
- "${API_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -259,7 +270,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: swagger
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab5}
|
||||
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab6}
|
||||
ports:
|
||||
- "${SWAGGER_BIND_ADDRESS}:8080"
|
||||
depends_on:
|
||||
@@ -289,7 +300,7 @@ services:
|
||||
|
||||
lab5-experiment:
|
||||
profiles: ["experiments"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab6}
|
||||
command: ["/app/lab5experiment"]
|
||||
environment:
|
||||
MESSENGER_CONFIG_PATH: /etc/messenger/messenger.yaml
|
||||
@@ -375,11 +386,12 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: prometheus
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab5}
|
||||
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab6}
|
||||
command:
|
||||
- --config.file=/etc/prometheus/prometheus.yml
|
||||
- --storage.tsdb.path=/prometheus
|
||||
- --storage.tsdb.retention.time=7d
|
||||
- --web.enable-remote-write-receiver
|
||||
volumes:
|
||||
- prometheus_data:/prometheus
|
||||
ports:
|
||||
@@ -398,6 +410,9 @@ services:
|
||||
- dispatcher
|
||||
- worker
|
||||
- s3
|
||||
- otel-collector
|
||||
- loki
|
||||
- tempo
|
||||
cpus: ${PROMETHEUS_CPUS}
|
||||
mem_limit: ${PROMETHEUS_MEMORY}
|
||||
restart: unless-stopped
|
||||
@@ -407,28 +422,98 @@ services:
|
||||
build:
|
||||
context: .
|
||||
target: grafana
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab5}
|
||||
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab6}
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
|
||||
GF_USERS_ALLOW_SIGN_UP: "false"
|
||||
GF_EXPLORE_ENABLED: "true"
|
||||
volumes:
|
||||
- grafana_data:/var/lib/grafana
|
||||
ports:
|
||||
- "${GRAFANA_BIND_ADDRESS}:3000"
|
||||
depends_on:
|
||||
- prometheus
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:3000/api/health || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
cpus: ${GRAFANA_CPUS}
|
||||
mem_limit: ${GRAFANA_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
loki:
|
||||
image: grafana/loki:3.7.9
|
||||
command: ["-config.file=/etc/loki/loki.yaml"]
|
||||
volumes:
|
||||
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
||||
- loki_data:/loki
|
||||
ports:
|
||||
- "${LOKI_BIND_ADDRESS}:3100"
|
||||
healthcheck:
|
||||
test: ["CMD", "/usr/bin/loki", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${LOKI_CPUS}
|
||||
mem_limit: ${LOKI_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
tempo:
|
||||
image: grafana/tempo:2.10.10
|
||||
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
||||
volumes:
|
||||
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
||||
- tempo_data:/var/tempo
|
||||
ports:
|
||||
- "${TEMPO_BIND_ADDRESS}:3200"
|
||||
healthcheck:
|
||||
test: ["CMD", "/tempo", "-version"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${TEMPO_CPUS}
|
||||
mem_limit: ${TEMPO_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [edge, monitoring]
|
||||
|
||||
otel-collector:
|
||||
image: otel/opentelemetry-collector-contrib:0.160.0
|
||||
command: ["--config=/etc/otelcol-contrib/collector.yaml"]
|
||||
volumes:
|
||||
- ./infra/otel/collector.yaml:/etc/otelcol-contrib/collector.yaml:ro
|
||||
expose:
|
||||
- "4317"
|
||||
- "4318"
|
||||
- "8888"
|
||||
- "8889"
|
||||
- "13133"
|
||||
depends_on:
|
||||
- loki
|
||||
- tempo
|
||||
healthcheck:
|
||||
test: ["CMD", "/otelcol-contrib", "components"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
cpus: ${OTEL_COLLECTOR_CPUS}
|
||||
mem_limit: ${OTEL_COLLECTOR_MEMORY}
|
||||
restart: unless-stopped
|
||||
networks: [application, data, monitoring]
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
valkey_data:
|
||||
redis_broker_data:
|
||||
prometheus_data:
|
||||
grafana_data:
|
||||
loki_data:
|
||||
tempo_data:
|
||||
s3_data:
|
||||
|
||||
networks:
|
||||
|
||||
Reference in New Issue
Block a user