Реализовать лабораторную 6 на 5: SLO и обнаружение инцидентов

This commit is contained in:
Ванек
2026-11-19 18:53:24 +03:00
parent 470d9d6f1e
commit 8dad129edf
127 changed files with 53496 additions and 61 deletions
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
python3 scripts/init_local.py
docker compose build "$@" api
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
if [[ ! -x .venv/bin/python ]]; then
echo "Создай .venv и установи requirements.txt по REPORT.md" >&2
exit 1
fi
./scripts/build.sh
mkdir -p evidence
.venv/bin/python -u student-tests/lab5_pipeline.py 2>&1 | sed -E 's/[[:blank:]]+$//' | tee evidence/lab5-checks.txt
+29
View File
@@ -0,0 +1,29 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
mkdir -p evidence
./scripts/build.sh
# Inherited grade-4 lab5 criteria, on a separate temporary project and fresh volumes.
LAB5_EVIDENCE_FILE=evidence/lab6-lab5-regression.json \
.venv/bin/python -u student-tests/lab5_pipeline.py 2>&1 | tee evidence/lab6-lab5-regression.txt
./scripts/start.sh
make check validate test PYTHON=.venv/bin/python 2>&1 | tee evidence/lab6-api-checks.txt
.venv/bin/python -m unittest discover -s student-tests -v 2>&1 | tee -a evidence/lab6-api-checks.txt
docker run --rm --entrypoint /bin/promtool -v "$project_dir:/work:ro" -w /work \
prom/prometheus:v3.4.1 test rules student-tests/promtool-slo.yaml | tee evidence/lab6-promtool.txt
# Only this short check enables the already inherited one-shot worker failpoint.
trap 'ENABLE_FAULT_INJECTION=0 docker compose up -d --no-deps worker >/dev/null 2>&1' EXIT
ENABLE_FAULT_INJECTION=1 docker compose up -d --no-deps worker
.venv/bin/python -u student-tests/lab6_observability.py 2>&1 | tee evidence/lab6-checks.txt
ENABLE_FAULT_INJECTION=0 docker compose up -d --no-deps worker
.venv/bin/python -u student-tests/verify_restart.py 2>&1 | tee evidence/lab6-restart.txt
.venv/bin/python - <<'PY'
import json
from research.observations import ROOT, spans, trace, wait
evidence=json.loads((ROOT/'evidence/lab6-checks.json').read_text())
trace_id=evidence['checks']['retry_trace_and_logs']['trace_id']
wait(lambda: len(spans(trace(trace_id))) >= 10)
print('PASS: trace remains queryable after restarting all telemetry stores with their volumes')
PY
.venv/bin/python -u research/slo.py 2>&1 | tee evidence/lab6-slo-research.txt
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
if [[ ! -x .venv/bin/python ]]; then
echo "Не хватает .venv" >&2
echo "python3 -m venv .venv" >&2
echo ".venv/bin/python -m pip install -r requirements.txt" >&2
exit 1
fi
export BASE_URL="${BASE_URL:-http://localhost:8080}"
echo "Проверяем сервер: $BASE_URL"
make check test PYTHON="$project_dir/.venv/bin/python" BASE_URL="$BASE_URL"
.venv/bin/python -m unittest discover -s student-tests -v
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -euo pipefail
base_url="${BASE_URL:-http://localhost:8080}"
suffix="$(date +%s)_$RANDOM"
# Только синтетические пароли; токены остаются в памяти shell и не выводятся.
password="Demo-password-$suffix"
register() {
local username="$1" display_name="$2"
jq -nc --arg u "$username" --arg d "$display_name" --arg p "$password" \
'{username:$u, display_name:$d, password:$p}' |
curl -fsS -H 'Content-Type: application/json' --data-binary @- "$base_url/api/v1/users" | jq -er '.id'
}
login() {
curl -fsS -X POST -u "$1:$password" "$base_url/api/v1/auth/sessions" | jq -er '.token'
}
alice_id="$(register "alice_$suffix" Алиса)"
bob_id="$(register "bob_$suffix" Боб)"
alice_token="$(login "alice_$suffix")"
bob_token="$(login "bob_$suffix")"
chat_id="$(jq -nc --arg a "$alice_id" --arg b "$bob_id" '{title:"Учебный чат",member_ids:[$a,$b]}' |
curl -fsS -H "Authorization: Bearer $alice_token" -H 'Content-Type: application/json' \
--data-binary @- "$base_url/api/v1/chats" | jq -er '.id')"
for token in "$alice_token" "$bob_token"; do
curl -fsS -H "Authorization: Bearer $token" -H 'Content-Type: application/json' \
-H "Idempotency-Key: demo-$RANDOM-$RANDOM" \
-d '{"text":"Привет из lab5! 👋"}' "$base_url/api/v1/chats/$chat_id/messages" | jq .
done
curl -fsS -H "Authorization: Bearer $bob_token" "$base_url/api/v1/chats/$chat_id/messages" | jq .
+62
View File
@@ -0,0 +1,62 @@
"""Создаёт локальное окружение и secrets; существующие значения сохраняет."""
from pathlib import Path
import json
import os
import re
import secrets
import shutil
from urllib.parse import quote
root = Path(__file__).resolve().parents[1]
if not (root / ".env").exists():
shutil.copyfile(root / ".env.example", root / ".env")
directory = root / "secrets"
directory.mkdir(exist_ok=True, mode=0o700)
directory.chmod(0o700)
password_file = directory / "postgres_password"
if not password_file.exists():
password_file.write_text(secrets.token_hex(24))
password = password_file.read_text().strip()
values = {
"database_url": f"postgresql://messenger:{quote(password, safe='')}@postgres:5432/messenger",
"cache_url": "redis://valkey:6379/0",
"broker_password": secrets.token_hex(24),
"s3_root_user": "root-" + secrets.token_hex(8),
"s3_root_password": secrets.token_urlsafe(32),
"s3_access_key_id": "api-" + secrets.token_hex(8),
"s3_secret_access_key": secrets.token_urlsafe(32),
}
for name, value in values.items():
path = directory / name
if not path.exists():
path.write_text(value)
broker_url = directory / "broker_url"
if not broker_url.exists():
broker_url.write_text("redis://:" + (directory / "broker_password").read_text().strip() + "@broker:6379/0")
broker_url.chmod(0o444)
dotenv = dict(line.split("=", 1) for line in (root / ".env").read_text().splitlines()
if line.strip() and not line.lstrip().startswith("#") and "=" in line)
bucket = os.environ.get("S3_BUCKET", dotenv.get("S3_BUCKET", "messenger-attachments"))
if re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket) is None:
raise ValueError("Недопустимое имя S3 bucket")
def local_secret(name):
return (directory / name).read_text().strip()
config = {"accounts": [{"id": "messenger-api", "displayName": "messenger-api"}], "identities": [
{"name": "bootstrap", "credentials": [{"accessKey": local_secret("s3_root_user"),
"secretKey": local_secret("s3_root_password")}], "actions": ["Admin"]},
{"name": "messenger-api", "account": {"id": "messenger-api", "displayName": "messenger-api"},
"credentials": [{"accessKey": local_secret("s3_access_key_id"), "secretKey": local_secret("s3_secret_access_key")}],
"actions": ["Read:" + bucket, "List:" + bucket, "Write:" + bucket + "/attachments/*"]},
]}
config_file = directory / "s3_config"
encoded_config = json.dumps(config)
if not config_file.exists() or config_file.read_text() != encoded_config:
temporary = directory / "s3_config.tmp"
temporary.write_text(encoded_config)
temporary.chmod(0o444)
temporary.replace(config_file)
# Каталог 0700 защищает файлы на хосте; 0444 позволяет USER 10001 читать bind-mounted secret.
for name in ("postgres_password", "s3_config", *values):
(directory / name).chmod(0o444)
print("Local .env and secrets are ready; values are not printed")
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
./scripts/build.sh
docker compose run --rm migrate
+90
View File
@@ -0,0 +1,90 @@
"""Управление репликами только текущего Compose project через Docker CLI."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
from urllib.request import urlopen
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(args, cwd=ROOT, text=True, timeout=120).strip()
def replicas():
ids = command("docker", "compose", "ps", "-q", "api").split()
return json.loads(command("docker", "inspect", *ids)) if ids else []
def base_url():
config = json.loads(command("docker", "compose", "config", "--format", "json"))
port = config["services"]["lb"]["ports"][0]["published"]
return os.environ.get("BASE_URL", "http://127.0.0.1:" + str(port)).rstrip("/")
def wait_ready():
url = base_url() + "/health/ready"
deadline = time.monotonic() + 90
seen = set()
while time.monotonic() < deadline:
current = replicas()
expected = {info["Config"]["Hostname"] for info in current}
if expected and all(info["State"].get("Health", {}).get("Status") == "healthy" for info in current):
try:
with urlopen(url, timeout=2) as response:
if response.status == 200:
seen.add(response.headers.get("X-Instance-Id"))
if expected <= seen:
print("API ready through LB:", ", ".join(sorted(expected)), flush=True)
return
except OSError:
pass
time.sleep(0.2)
raise RuntimeError("Не все работающие API-реплики готовы через LB за 90 секунд")
def stop_one():
target = os.environ.get("TARGET_INSTANCE_ID")
if not target:
raise RuntimeError("Задайте TARGET_INSTANCE_ID из X-Instance-Id")
current = replicas()
matches = [info for info in current if info["Config"]["Hostname"] == target]
if len(current) < 2 or len(matches) != 1:
raise RuntimeError("Нужны минимум две работающие реплики и ровно одна выбранная")
selected = matches[0]
command("docker", "stop", "--time", "8", selected["Id"])
print("Stopped API:", target, "container=" + selected["Id"], flush=True)
def rolling():
wait_ready()
originals = replicas()
count = len(originals)
if count < 2:
raise RuntimeError("Для поочерёдной замены нужны минимум две API-реплики")
for old in originals:
active = replicas()
healthy = [info for info in active if info["Id"] != old["Id"]
and info["State"].get("Health", {}).get("Status") == "healthy"]
if not healthy:
raise RuntimeError("Нет готовой реплики, которая продолжит обслуживать запросы")
print("Replacing API:", old["Config"]["Hostname"], flush=True)
command("docker", "stop", "--time", "8", old["Id"])
command("docker", "rm", old["Id"])
# --no-recreate сохраняет остальные процессы; Compose создаёт только недостающий.
command("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={count}", "api")
wait_ready()
assert len(replicas()) == count
final = replicas()
assert not ({info["Id"] for info in originals} & {info["Id"] for info in final})
print("PASS: all API containers replaced sequentially; stores and LB kept running", flush=True)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("action", choices=["wait", "stop", "rolling"])
action = parser.parse_args().action
{"wait": wait_ready, "stop": stop_one, "rolling": rolling}[action]()
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
./scripts/build.sh
mkdir -p evidence
python3 -u research/run.py 2>&1 | tee evidence/research.txt
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
python3 scripts/wait-observability.py
mkdir -p evidence
.venv/bin/python -u research/slo.py 2>&1 | tee evidence/lab6-slo-research.txt
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
docker compose restart --timeout 8 postgres valkey s3-storage s3 broker api lb cleanup publisher worker otel-collector prometheus loki tempo grafana
./scripts/wait-ready.sh
python3 scripts/wait-observability.py
+7
View File
@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
# Сначала собираем образ и применяем только обратно совместимые миграции.
./scripts/migrate.sh
exec python3 scripts/replicas.py rolling
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
# Локальный стенд: HAProxy, две API-реплики, PostgreSQL и Valkey.
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
exec "$project_dir/scripts/start.sh"
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
docker compose start worker
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
python3 scripts/init_local.py
docker compose up -d --build lb cleanup publisher worker otel-collector prometheus grafana
./scripts/wait-ready.sh
python3 scripts/wait-observability.py
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
exec python3 scripts/replicas.py stop
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
docker compose stop --timeout 8 worker
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
docker compose down --timeout 10
echo "Контейнеры удалены; именованные volumes сохранены"
+26
View File
@@ -0,0 +1,26 @@
"""Readiness of provisioned telemetry, queried through the local Grafana proxy."""
from pathlib import Path
import sys
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from research.observations import get, proxy, query, wait
def ready():
assert get('/api/health')['database'] == 'ok'
assert get('/api/dashboards/uid/messenger-lab6')['dashboard']['uid'] == 'messenger-lab6'
assert len(get('/api/datasources')) == 3
assert len(query('up == 1')) == 3
proxy('loki', '/loki/api/v1/labels')
proxy('tempo', '/api/search', limit=1)
return True
if __name__ == '__main__':
# Assertion failures are readiness failures, not successful probes.
def probe():
try:
return ready()
except (AssertionError, OSError, ValueError):
return False
wait(probe, timeout=90)
print('PASS: Collector scrape, stores, Grafana datasources and dashboard are ready')
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
project_dir="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$project_dir"
exec python3 scripts/replicas.py wait