Files
mtusi2026lab3-fork/scripts/collect_evidence.py

175 lines
10 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Прогон всех проверок lab3 на запущенном compose-стенде с записью в evidence/.
python scripts/collect_evidence.py # ~8-10 минут
python scripts/collect_evidence.py --quick # без kill-серии и нагрузки
Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`,
`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики,
перезапускает хранилища (volumes сохраняются) и делает rolling update
lab3 -> lab3.1 -> lab3 (откат), поэтому запускайте его только на учебном стенде.
Токены и пароли в файлы не пишутся (тесты держат их в памяти).
"""
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
EV = ROOT / "evidence"
PY = sys.executable or "python"
WIN = os.name == "nt"
HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh"))
HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh"))
BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080")
RESULTS = []
try:
sys.stdout.reconfigure(errors="replace")
except AttributeError:
pass
def stamp():
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
def step(title, cmd, out_file, append=False, env=None, timeout=900):
"""Выполнить команду, сохранить «$ команда + вывод» в evidence/<out_file>."""
print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True)
started = time.monotonic()
try:
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
timeout=timeout, env=dict(os.environ, **(env or {})),
encoding="utf-8", errors="replace")
output, rc = proc.stdout, proc.returncode
except subprocess.TimeoutExpired as exc:
output, rc = (exc.stdout or "") + "\nTIMEOUT", 124
elapsed = time.monotonic() - started
shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "")
text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip())
path = EV / out_file
with path.open("a" if append else "w", encoding="utf-8") as f:
f.write(text + ("\n" if append else ""))
print(output.rstrip()[-1500:], flush=True)
RESULTS.append((title, rc, out_file))
return rc, output
def instances_through_lb(n=20):
"""Одна сессия, n запросов через LB -> какие реплики отвечали."""
import base64
import uuid
name = "inst_" + uuid.uuid4().hex[:12]
pw = "Inst-" + uuid.uuid4().hex
body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode()
urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10)
cred = base64.b64encode((name + ":" + pw).encode()).decode()
sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST",
headers={"Authorization": "Basic " + cred}), timeout=10).read())
lines = []
for i in range(n):
with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r:
lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % (
i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id")))
return lines
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--quick", action="store_true")
args = ap.parse_args()
EV.mkdir(exist_ok=True)
(EV / "ops.log").write_text("", encoding="utf-8")
step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt")
step("Compose config", ["docker", "compose", "--env-file", ".env", "config", "--quiet"], "check.txt", append=True)
step("Стенд: контейнеры", ["docker", "compose", "--env-file", ".env", "ps", "-a"], "stand.txt")
step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True)
step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True)
step("Стенд: опубликованные порты (только lb)", ["docker", "compose", "--env-file", ".env", "ps",
"--format", "table {{.Service}}\t{{.Ports}}"], "stand.txt", append=True)
lines = instances_through_lb()
(EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n"
% (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})),
encoding="utf-8")
print("\n".join(lines))
step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt")
step("Собственные тесты (HTTP через LB)", [PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt")
step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"],
"student-tests.txt", append=True)
step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"],
"student-tests.txt", append=True)
# Публичный тест останавливает реплику, обслужившую его первое сообщение.
# Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь
# лишними запросами, пока цель не окажется другой (README: «повторить для второй»).
targets = []
for i in range(1, 5):
if len(set(targets)) >= 2:
break
for _ in range(i - 1):
urlopen(BASE + "/health/live", timeout=5).read()
step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1)
step("Failover #%d: make test-failover" % i,
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE],
"failover.txt", append=True)
step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"],
"failover.txt", append=True)
step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True)
stops = [l for l in (EV / "ops.log").read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l]
if stops:
targets.append(stops[-1].split("target=")[1].split()[0])
with (EV / "failover.txt").open("a", encoding="utf-8") as f:
f.write("# остановленные публичным тестом реплики: %s\n" % targets)
step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--output", "evidence/failover_series.json"], "failover_series.txt")
if not args.quick:
step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--mode", "kill", "--output", "evidence/failover_series_kill.json"],
"failover_series.txt", append=True)
step("Rolling update lab3 -> lab3.1 под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"],
"rolling_update.txt", env={"APP_VERSION": "lab3.1"})
step("Rolling update обратно lab3.1 -> lab3 (откат) под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"],
"rolling_update.txt", append=True, env={"APP_VERSION": "lab3"})
step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True)
step("Persistence: контейнеры до", ["docker", "compose", "--env-file", ".env", "ps", "-a", "--format",
"table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"], "persistence.txt")
step("Persistence: make test-persistence",
[PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE],
"persistence.txt", append=True)
step("Persistence: контейнеры после (Status = Up N seconds)", ["docker", "compose", "--env-file", ".env", "ps", "-a",
"--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"],
"persistence.txt", append=True)
if not args.quick:
step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30",
"--concurrency", "8", "--output", "evidence/load.json"], "load.txt")
step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True)
step("Лог LB (последние строки, без секретов)", ["docker", "compose", "--env-file", ".env", "logs", "--no-color",
"--tail", "60", "lb"], "lb_log_sample.txt")
summary = ["# %s" % stamp()] + ["%-4s %-60s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS]
(EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8")
print("\n" + "\n".join(summary))
sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1)
if __name__ == "__main__":
main()