forked from mtusi2026/mtusi2026lab6
feat(lab6): add observability and SLO burn-rate alerts
This commit is contained in:
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env sh
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Keep named volumes (business data and telemetry history) intact. Only this
|
||||
# Compose project's containers/network and the five images built by lab6 are
|
||||
# removed. Pulled third-party images and resources from other labs are never
|
||||
# selected by this script.
|
||||
docker compose down --remove-orphans
|
||||
|
||||
for image in \
|
||||
mtusi2026lab6-app:latest \
|
||||
mtusi2026lab6-web:latest \
|
||||
mtusi2026lab6-load-balancer:latest \
|
||||
mtusi2026lab6-minio:RELEASE.2025-10-15T17-29-55Z \
|
||||
mtusi2026lab6-mc:RELEASE.2025-08-13T08-35-41Z
|
||||
do
|
||||
if docker image inspect "$image" >/dev/null 2>&1; then
|
||||
docker image rm "$image"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "lab6 containers are stopped; local lab6 images removed; named volumes preserved"
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
secrets_dir="$project_root/secrets"
|
||||
|
||||
command -v openssl >/dev/null 2>&1 || {
|
||||
printf '%s\n' "openssl is required to generate local secrets" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
umask 077
|
||||
mkdir -p "$secrets_dir"
|
||||
chmod 700 "$secrets_dir"
|
||||
|
||||
write_if_missing() {
|
||||
destination=$1
|
||||
value=$2
|
||||
if [ ! -e "$destination" ]; then
|
||||
printf '%s' "$value" > "$destination"
|
||||
fi
|
||||
chmod 600 "$destination"
|
||||
}
|
||||
|
||||
random_hex() {
|
||||
openssl rand -hex "$1"
|
||||
}
|
||||
|
||||
write_if_missing "$secrets_dir/postgres_password.txt" "$(random_hex 24)"
|
||||
postgres_password=$(cat "$secrets_dir/postgres_password.txt")
|
||||
write_if_missing "$secrets_dir/database_url.txt" \
|
||||
"postgres://messenger:${postgres_password}@postgres:5432/messenger?sslmode=disable"
|
||||
write_if_missing "$secrets_dir/cursor_signing_key.txt" "$(random_hex 32)"
|
||||
write_if_missing "$secrets_dir/minio_root_user.txt" "lab6-root"
|
||||
write_if_missing "$secrets_dir/minio_root_password.txt" "$(random_hex 24)"
|
||||
write_if_missing "$secrets_dir/s3_access_key_id.txt" "lab6-api"
|
||||
write_if_missing "$secrets_dir/s3_secret_access_key.txt" "$(random_hex 24)"
|
||||
write_if_missing "$secrets_dir/broker_password.txt" "$(random_hex 24)"
|
||||
broker_password=$(cat "$secrets_dir/broker_password.txt")
|
||||
write_if_missing "$secrets_dir/broker_url.txt" "amqp://messenger:${broker_password}@broker:5672/"
|
||||
|
||||
printf '%s\n' "Local secret files are present with mode 600; existing values were not overwritten."
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_root"
|
||||
|
||||
env_file=${ENV_FILE:-.env}
|
||||
compose() {
|
||||
if [ -f "$env_file" ]; then
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# Останавливаем consumers до рестарта устойчивых зависимостей. PostgreSQL,
|
||||
# MinIO и RabbitMQ используют named volumes; Redis остаётся rebuildable cache.
|
||||
compose stop worker dispatcher
|
||||
compose restart postgres redis minio broker
|
||||
compose up -d --wait postgres redis minio broker
|
||||
|
||||
# Повторный bootstrap приватного bucket/policy/user обязан быть идемпотентным.
|
||||
compose run --rm minio-bootstrap >/dev/null
|
||||
|
||||
# Dispatcher восстановит все DB jobs без publish marker; worker продолжит
|
||||
# durable queue. Фоновые процессы не имеют HTTP readiness endpoint.
|
||||
compose up -d --no-deps dispatcher worker
|
||||
|
||||
# up восстанавливает желаемое число реплик, включая ранее вручную остановленную.
|
||||
compose up -d --no-deps api
|
||||
compose restart api
|
||||
compose up -d --wait --no-deps api
|
||||
|
||||
# Статический frontend возвращается до публичной точки входа.
|
||||
compose restart frontend
|
||||
compose up -d --wait --no-deps frontend
|
||||
|
||||
# Публичная точка входа перезапускается последней, когда API и frontend готовы.
|
||||
compose restart load-balancer
|
||||
compose up -d --wait --no-deps load-balancer
|
||||
|
||||
# Telemetry storage uses separate named volumes; restart the signal path without
|
||||
# deleting history. Business readiness never depends on these services.
|
||||
compose restart loki tempo otel-collector prometheus grafana
|
||||
compose up -d --wait --no-deps loki tempo otel-collector prometheus grafana
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
project_root=$(CDPATH='' cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_root"
|
||||
|
||||
env_file=${ENV_FILE:-.env}
|
||||
matched_container=
|
||||
|
||||
compose() {
|
||||
if [ -f "$env_file" ]; then
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
else
|
||||
docker compose "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
for container_id in $(compose ps -a -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container_id")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
if [ -n "$matched_container" ]; then
|
||||
printf '%s\n' "multiple API containers match instance $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
matched_container=$container_id
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$matched_container" ]; then
|
||||
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# A manual docker stop suppresses the container restart policy until the next
|
||||
# explicit start, so the selected replica remains down for the whole test.
|
||||
docker stop --time 5 "$matched_container" >/dev/null
|
||||
|
||||
printf '%s\n' "stopped API instance $TARGET_INSTANCE_ID"
|
||||
Reference in New Issue
Block a user