forked from mtusi2026/mtusi2026lab6
417 lines
14 KiB
YAML
417 lines
14 KiB
YAML
# Lab 6: lab 5 stack + observability (OTel Collector -> Prometheus / Loki / Tempo -> Grafana).
|
|
name: mtusi-lab6
|
|
|
|
x-otel-env: &otel-env # SDK exporter settings shared by api, worker and dispatcher (service.name is set in code)
|
|
OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318
|
|
OTEL_EXPORTER_OTLP_PROTOCOL: http/protobuf
|
|
OTEL_RESOURCE_ATTRIBUTES: deployment.environment.name=lab
|
|
SERVICE_VERSION: "1.6.0"
|
|
|
|
services:
|
|
postgres:
|
|
image: postgres:17.6-alpine
|
|
environment:
|
|
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
|
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
|
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
volumes:
|
|
- postgres_data:/var/lib/postgresql/data
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 20
|
|
cpus: 1.0
|
|
mem_limit: 768m
|
|
restart: unless-stopped
|
|
|
|
redis:
|
|
image: redis:7.4.2-alpine
|
|
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
|
volumes:
|
|
- redis_data:/data
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 5s
|
|
timeout: 3s
|
|
retries: 20
|
|
cpus: 0.25
|
|
mem_limit: 256m
|
|
restart: unless-stopped
|
|
|
|
s3:
|
|
image: pgsty/silo:RELEASE.2026-10-15T00-00-00Z
|
|
command: ["server", "/data", "--address", ":9000"]
|
|
environment:
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
|
MINIO_BROWSER: "off" # admin web console disabled; no ports are published at all
|
|
volumes:
|
|
- s3_data:/data
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD", "mc", "ready", "local"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 20
|
|
cpus: 1.0
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
s3_init:
|
|
image: pgsty/silo:RELEASE.2026-10-15T00-00-00Z
|
|
entrypoint: ["/bin/sh", "/infra/s3-init.sh"]
|
|
environment:
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
|
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
|
SWEEPER_S3_ACCESS_KEY_ID: ${SWEEPER_S3_ACCESS_KEY_ID:?Set SWEEPER_S3_ACCESS_KEY_ID in .env}
|
|
SWEEPER_S3_SECRET_ACCESS_KEY: ${SWEEPER_S3_SECRET_ACCESS_KEY:?Set SWEEPER_S3_SECRET_ACCESS_KEY in .env}
|
|
volumes:
|
|
- ./infra/s3-init.sh:/infra/s3-init.sh:ro
|
|
networks: [backend]
|
|
depends_on:
|
|
s3:
|
|
condition: service_started
|
|
restart: "no"
|
|
|
|
kafka:
|
|
image: apache/kafka:3.9.0
|
|
environment:
|
|
KAFKA_NODE_ID: "1"
|
|
KAFKA_PROCESS_ROLES: broker,controller
|
|
KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
|
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
|
|
KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
|
|
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
|
|
KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
|
|
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
|
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
|
|
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: "1"
|
|
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: "1"
|
|
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: "0"
|
|
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false"
|
|
KAFKA_LOG_DIRS: /var/lib/kafka/data
|
|
KAFKA_HEAP_OPTS: "-Xms256m -Xmx512m"
|
|
volumes:
|
|
- kafka_data:/var/lib/kafka/data
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092 >/dev/null 2>&1"]
|
|
interval: 5s
|
|
timeout: 10s
|
|
retries: 30
|
|
start_period: 15s
|
|
cpus: 1.0
|
|
mem_limit: 1g
|
|
restart: unless-stopped
|
|
|
|
kafka_init:
|
|
image: apache/kafka:3.9.0
|
|
entrypoint: ["/bin/sh", "-c"]
|
|
command:
|
|
- |
|
|
set -e
|
|
T=/opt/kafka/bin/kafka-topics.sh
|
|
$$T --bootstrap-server kafka:9092 --create --if-not-exists --topic $${JOBS_TOPIC} --partitions 3 --replication-factor 1 --config min.insync.replicas=1 --config retention.ms=604800000
|
|
$$T --bootstrap-server kafka:9092 --list
|
|
environment:
|
|
JOBS_TOPIC: ${JOBS_TOPIC:-attachment-jobs}
|
|
networks: [backend]
|
|
depends_on:
|
|
kafka:
|
|
condition: service_healthy
|
|
restart: "no"
|
|
|
|
migrate:
|
|
build: .
|
|
image: mtusi-lab6-api:1.0.0
|
|
command: ["python", "scripts/migrate.py"]
|
|
environment:
|
|
PGHOST: postgres
|
|
PGPORT: "5432"
|
|
PGDATABASE: ${POSTGRES_DB:-messenger}
|
|
PGUSER: ${POSTGRES_USER:-messenger}
|
|
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
networks: [backend]
|
|
depends_on:
|
|
postgres:
|
|
condition: service_healthy
|
|
restart: "no"
|
|
cpus: 0.5
|
|
mem_limit: 384m
|
|
|
|
api:
|
|
build: .
|
|
image: mtusi-lab6-api:1.0.0
|
|
environment:
|
|
APP_HOST: 0.0.0.0
|
|
APP_PORT: "8080"
|
|
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
|
DB_POOL_MIN_SIZE: ${DB_POOL_MIN_SIZE:-1}
|
|
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-10}
|
|
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-5}
|
|
PGHOST: postgres
|
|
PGPORT: "5432"
|
|
PGDATABASE: ${POSTGRES_DB:-messenger}
|
|
PGUSER: ${POSTGRES_USER:-messenger}
|
|
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
REDIS_URL: redis://redis:6379/0
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
|
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
IMAGE_DECODE_CONCURRENCY: ${IMAGE_DECODE_CONCURRENCY:-1}
|
|
<<: *otel-env
|
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
|
deploy:
|
|
replicas: 2
|
|
expose: ["8080"]
|
|
networks: [backend]
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
redis:
|
|
condition: service_healthy
|
|
s3_init:
|
|
condition: service_completed_successfully
|
|
healthcheck:
|
|
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 10
|
|
start_period: 20s
|
|
stop_grace_period: 15s
|
|
cpus: 1.0
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
worker:
|
|
build: .
|
|
image: mtusi-lab6-api:1.0.0
|
|
command: ["python", "-m", "src.worker"]
|
|
environment:
|
|
PGHOST: postgres
|
|
PGPORT: "5432"
|
|
PGDATABASE: ${POSTGRES_DB:-messenger}
|
|
PGUSER: ${POSTGRES_USER:-messenger}
|
|
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
|
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
|
|
JOBS_TOPIC: ${JOBS_TOPIC:-attachment-jobs}
|
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
|
LEASE_GRACE_SECONDS: ${LEASE_GRACE_SECONDS:-15}
|
|
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
|
JOB_BACKOFF_CAP_SECONDS: ${JOB_BACKOFF_CAP_SECONDS:-60}
|
|
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
|
IMG_MEMORY_MB: ${IMG_MEMORY_MB:-1024}
|
|
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
|
FAULT_WORKER_EXIT_AFTER_S3: ${FAULT_WORKER_EXIT_AFTER_S3:-0}
|
|
<<: *otel-env
|
|
networks: [backend]
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
kafka_init:
|
|
condition: service_completed_successfully
|
|
s3_init:
|
|
condition: service_completed_successfully
|
|
stop_grace_period: 30s
|
|
cpus: 1.0
|
|
mem_limit: 1g
|
|
restart: unless-stopped
|
|
|
|
dispatcher:
|
|
build: .
|
|
image: mtusi-lab6-api:1.0.0
|
|
command: ["python", "-m", "src.dispatcher"]
|
|
environment:
|
|
PGHOST: postgres
|
|
PGPORT: "5432"
|
|
PGDATABASE: ${POSTGRES_DB:-messenger}
|
|
PGUSER: ${POSTGRES_USER:-messenger}
|
|
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
KAFKA_BOOTSTRAP_SERVERS: kafka:9092
|
|
JOBS_TOPIC: ${JOBS_TOPIC:-attachment-jobs}
|
|
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
|
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
|
LEASE_GRACE_SECONDS: ${LEASE_GRACE_SECONDS:-15}
|
|
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
|
JOB_BACKOFF_CAP_SECONDS: ${JOB_BACKOFF_CAP_SECONDS:-60}
|
|
FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH: ${FAULT_DISPATCHER_EXIT_BEFORE_PUBLISH:-0}
|
|
<<: *otel-env
|
|
networks: [backend]
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
kafka_init:
|
|
condition: service_completed_successfully
|
|
s3_init:
|
|
condition: service_completed_successfully
|
|
cpus: 0.25
|
|
mem_limit: 192m
|
|
restart: unless-stopped
|
|
|
|
s3_sweeper:
|
|
build: .
|
|
image: mtusi-lab6-api:1.0.0
|
|
command: ["python", "scripts/s3_sweep.py", "--loop", "--apply"]
|
|
environment:
|
|
PGHOST: postgres
|
|
PGPORT: "5432"
|
|
PGDATABASE: ${POSTGRES_DB:-messenger}
|
|
PGUSER: ${POSTGRES_USER:-messenger}
|
|
PGPASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
|
S3_ENDPOINT: http://s3:9000
|
|
S3_REGION: ${S3_REGION:-us-east-1}
|
|
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
|
S3_ACCESS_KEY_ID: ${SWEEPER_S3_ACCESS_KEY_ID:?Set SWEEPER_S3_ACCESS_KEY_ID in .env}
|
|
S3_SECRET_ACCESS_KEY: ${SWEEPER_S3_SECRET_ACCESS_KEY:?Set SWEEPER_S3_SECRET_ACCESS_KEY in .env}
|
|
SWEEP_GRACE_SECONDS: ${SWEEP_GRACE_SECONDS:-3600}
|
|
SWEEP_INTERVAL_SECONDS: ${SWEEP_INTERVAL_SECONDS:-600}
|
|
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
|
networks: [backend]
|
|
depends_on:
|
|
migrate:
|
|
condition: service_completed_successfully
|
|
s3_init:
|
|
condition: service_completed_successfully
|
|
cpus: 0.25
|
|
mem_limit: 128m
|
|
restart: unless-stopped
|
|
|
|
# ---------------- Lab 6: observability ----------------
|
|
otel-collector:
|
|
image: otel/opentelemetry-collector-contrib:0.131.0
|
|
command: ["--config=/etc/otelcol/collector.yaml"]
|
|
volumes:
|
|
- ./infra/otel/collector.yaml:/etc/otelcol/collector.yaml:ro
|
|
networks: [backend] # OTLP :4318/:4317 and self-metrics :8888 are NOT published to the host
|
|
# No healthcheck: the image has no shell/curl. Liveness extension health_check listens on :13133 inside the network.
|
|
cpus: 0.5
|
|
mem_limit: 256m # memory_limiter in collector.yaml stays below this (limit_mib: 192)
|
|
restart: unless-stopped
|
|
|
|
prometheus:
|
|
image: prom/prometheus:v3.5.0
|
|
command:
|
|
- --config.file=/etc/prometheus/prometheus.yml
|
|
- --storage.tsdb.path=/prometheus
|
|
- --storage.tsdb.retention.time=30d
|
|
- --web.enable-remote-write-receiver # the Collector pushes metrics here (prometheusremotewrite)
|
|
volumes:
|
|
- ./infra/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
|
|
- ./infra/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
|
|
- prometheus_data:/prometheus
|
|
networks: [backend]
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "--spider", "http://localhost:9090/-/ready"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 10
|
|
cpus: 0.5
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
loki:
|
|
image: grafana/loki:3.5.3
|
|
command: ["-config.file=/etc/loki/loki.yaml"]
|
|
volumes:
|
|
- ./infra/loki/loki.yaml:/etc/loki/loki.yaml:ro
|
|
- loki_data:/loki
|
|
networks: [backend]
|
|
cpus: 0.5
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
tempo:
|
|
image: grafana/tempo:2.8.2
|
|
command: ["-config.file=/etc/tempo/tempo.yaml"]
|
|
user: "0" # named volume must be writable on first start (learning stand; lab 7 hardens this)
|
|
volumes:
|
|
- ./infra/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
|
|
- tempo_data:/var/tempo
|
|
networks: [backend]
|
|
cpus: 0.5
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
grafana:
|
|
image: grafana/grafana:12.1.1
|
|
environment:
|
|
GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin}
|
|
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?Set GRAFANA_ADMIN_PASSWORD in .env}
|
|
GF_USERS_ALLOW_SIGN_UP: "false"
|
|
GF_ANALYTICS_REPORTING_ENABLED: "false"
|
|
GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
|
|
ports:
|
|
- "127.0.0.1:${GRAFANA_PORT:-3000}:3000" # the only observability port on the host, loopback only
|
|
volumes:
|
|
- ./infra/grafana/provisioning:/etc/grafana/provisioning:ro
|
|
- ./infra/grafana/dashboards:/var/lib/grafana/dashboards:ro
|
|
- grafana_data:/var/lib/grafana
|
|
networks: [frontend, backend]
|
|
depends_on:
|
|
prometheus:
|
|
condition: service_healthy
|
|
loki:
|
|
condition: service_started
|
|
tempo:
|
|
condition: service_started
|
|
healthcheck:
|
|
test: ["CMD", "wget", "-q", "--spider", "http://localhost:3000/api/health"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 10
|
|
cpus: 0.5
|
|
mem_limit: 512m
|
|
restart: unless-stopped
|
|
|
|
load_balancer:
|
|
image: haproxy:3.2.5-alpine
|
|
ports:
|
|
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
|
volumes:
|
|
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
|
networks: [frontend, backend]
|
|
depends_on:
|
|
api:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 3
|
|
cpus: 0.5
|
|
mem_limit: 128m
|
|
restart: unless-stopped
|
|
|
|
networks:
|
|
frontend:
|
|
driver: bridge
|
|
backend:
|
|
driver: bridge
|
|
internal: true # no route to/from the host or internet: S3, PostgreSQL, Redis, Kafka, API and the telemetry stack are reachable only inside
|
|
volumes:
|
|
postgres_data:
|
|
redis_data:
|
|
s3_data:
|
|
kafka_data:
|
|
prometheus_data:
|
|
loki_data:
|
|
tempo_data:
|
|
grafana_data:
|