Files
mtusi2026lab6-fork/scripts/collect_evidence.py

231 lines
15 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Прогон всех проверок lab5 (и унаследованных lab2-4) на запущенном compose-стенде с записью в evidence/.
python scripts/collect_evidence.py # ~18-22 минуты
python scripts/collect_evidence.py --quick # ~10-12 минут: без kill-серии, rolling update и нагрузки
Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`,
`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики,
перезапускает хранилища и S3 (volumes сохраняются), останавливает S3 и делает
rolling update lab5 -> lab5.1 -> lab5 (откат), останавливает worker/broker, поэтому запускайте его только на учебном стенде.
Токены и пароли в файлы не пишутся (тесты держат их в памяти).
"""
import argparse
import json
import os
import subprocess
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
from urllib.request import Request, urlopen
ROOT = Path(__file__).resolve().parents[1]
EV = ROOT / "evidence"
PY = sys.executable or "python"
WIN = os.name == "nt"
HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh"))
HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh"))
BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080")
RESULTS = []
try:
sys.stdout.reconfigure(errors="replace")
except AttributeError:
pass
def stamp():
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
def step(title, cmd, out_file, append=False, env=None, timeout=900):
"""Выполнить команду, сохранить «$ команда + вывод» в evidence/<out_file>."""
print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True)
started = time.monotonic()
try:
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
timeout=timeout, env=dict(os.environ, **(env or {})),
encoding="utf-8", errors="replace")
output, rc = proc.stdout, proc.returncode
except subprocess.TimeoutExpired as exc:
output, rc = (exc.stdout or "") + "\nTIMEOUT", 124
elapsed = time.monotonic() - started
shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "")
text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip())
path = EV / out_file
with path.open("a" if append else "w", encoding="utf-8") as f:
f.write(text + ("\n" if append else ""))
print(output.rstrip()[-1500:], flush=True)
RESULTS.append((title, rc, out_file))
return rc, output
def instances_through_lb(n=20):
"""Одна сессия, n запросов через LB -> какие реплики отвечали."""
import base64
import uuid
name = "inst_" + uuid.uuid4().hex[:12]
pw = "Inst-" + uuid.uuid4().hex
body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode()
urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10)
cred = base64.b64encode((name + ":" + pw).encode()).decode()
sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST",
headers={"Authorization": "Basic " + cred}), timeout=10).read())
lines = []
for i in range(n):
with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r:
lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % (
i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id")))
return lines
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--quick", action="store_true", help="без SIGKILL-серии, rolling update и нагрузки")
args = ap.parse_args()
EV.mkdir(exist_ok=True)
(EV / "ops.log").write_text("", encoding="utf-8")
dc = ["docker", "compose", "--env-file", ".env"]
step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt")
step("Compose config", dc + ["config", "--quiet"], "check.txt", append=True)
step("Стенд: контейнеры", dc + ["ps", "-a"], "stand.txt")
step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True)
step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True)
step("Стенд: опубликованные порты (только lb)", dc + ["ps", "--format", "table {{.Service}}\t{{.Ports}}"],
"stand.txt", append=True)
step("Образ SeaweedFS (закреплённая версия)", dc + ["images", "s3"], "stand.txt", append=True)
step("lab5: broker (Redis AOF) и worker", dc + ["ps", "broker", "worker"], "stand.txt", append=True)
step("lab5: конфигурация durability брокера", dc + ["exec", "-T", "broker", "sh", "-c",
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET appendfsync; "
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET maxmemory-policy"],
"stand.txt", append=True)
# Bucket bootstrap: первый запуск был при up; повтор — no-op.
step("S3 bootstrap: лог первого запуска s3-init", dc + ["logs", "--no-color", "s3-init"], "s3_bootstrap.txt")
step("S3 bootstrap: повторный запуск (идемпотентность)", dc + ["run", "--rm", "-T", "s3-init"],
"s3_bootstrap.txt", append=True)
lines = instances_through_lb()
(EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n"
% (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})),
encoding="utf-8")
print("\n".join(lines))
step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt")
step("Собственные тесты (HTTP через LB, включая test_attachments.py)",
[PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt")
step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"],
"student-tests.txt", append=True)
step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"],
"student-tests.txt", append=True)
step("Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*)",
[PY, "scripts/ops.py", "in-network", "consistency"], "consistency.txt")
step("Сверка БД <-> S3 (dry-run)", [PY, "scripts/ops.py", "reconcile"], "reconcile.txt")
step("Приватный бакет вне API, права ключей, закрытые админ-порты", [PY, "scripts/ops.py", "s3-check"],
"s3_private.txt")
step("На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes",
[PY, "scripts/ops.py", "api-fs"], "s3_private.txt", append=True)
step("API: файлы в /tmp (tmpfs) после загрузок",
dc + ["exec", "-T", "api", "sh", "-c", "df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l)"],
"s3_private.txt", append=True)
step("Пик памяти: 10 одновременных загрузок по 9.5 MiB",
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "10", "--rounds", "3",
"--docker", "--label", "c=10, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c10.json"],
"upload_memory.txt")
step("Пик памяти: 20 одновременных загрузок",
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "20", "--rounds", "2",
"--docker", "--label", "c=20, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c20.json"],
"upload_memory.txt", append=True)
# Публичный тест останавливает реплику, обслужившую его первое сообщение.
# Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь
# лишними запросами, пока цель не окажется другой (README: «повторить для второй»).
targets = []
for i in range(1, 5):
if len(set(targets)) >= 2:
break
for _ in range(i - 1):
urlopen(BASE + "/health/live", timeout=5).read()
ops_log = EV / "ops.log"
stops_before = sum(1 for l in ops_log.read_text(encoding="utf-8").splitlines()
if "stop-one: mode=" in l) if ops_log.exists() else 0
step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1)
step("Failover #%d: make test-failover (вкл. скачивание файла через другую реплику)" % i,
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE],
"failover.txt", append=True)
step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"],
"failover.txt", append=True)
step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True)
stops = [l for l in ops_log.read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l] \
if ops_log.exists() else []
# Только остановка из ЭТОЙ попытки: если тест упал до hook'а, новой строки нет,
# и цель прошлой попытки не должна засчитываться повторно.
if len(stops) > stops_before:
targets.append(stops[-1].split("target=")[1].split()[0])
with (EV / "failover.txt").open("a", encoding="utf-8") as f:
f.write("# остановленные публичным тестом реплики: %s\n" % targets)
step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--output", "evidence/failover_series.json"], "failover_series.txt")
if not args.quick:
step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE,
"--mode", "kill", "--output", "evidence/failover_series_kill.json"],
"failover_series.txt", append=True)
step("Недоступность S3 и восстановление", [PY, "student-tests/s3_outage.py", "--base-url", BASE,
"--output", "evidence/s3_outage.json"], "s3_outage.txt")
# lab5: очередь и worker
step("Варианты изображений по фактическим байтам (test_pipeline.py)",
[PY, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_pipeline.py", "-v"],
"pipeline.txt", env={"BASE_URL": BASE})
step("Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready",
[PY, "student-tests/worker_outage.py", "--base-url", BASE, "--output", "evidence/worker_outage.json"],
"worker_outage.txt")
step("Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)",
[PY, "scripts/ops.py", "pipeline-faults"], "pipeline_faults.txt", timeout=1200)
step("Очередь после сценариев: задания, outbox, DLQ, lag", [PY, "scripts/ops.py", "jobs", "stats"],
"pipeline_faults.txt", append=True)
step("Реестр failed (DLQ)", [PY, "scripts/ops.py", "jobs", "failed", "--limit", "10"], "pipeline_faults.txt",
append=True)
step("Worker: последние строки лога", dc + ["logs", "--no-color", "--tail", "80", "worker"], "worker_log_sample.txt")
if not args.quick:
step("Rolling update lab5 -> lab5.1 под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"],
"rolling_update.txt", env={"APP_VERSION": "lab5.1"})
step("Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой",
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"],
"rolling_update.txt", append=True, env={"APP_VERSION": "lab5"})
step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True)
fmt = ["ps", "-a", "--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"]
step("Persistence: контейнеры до", dc + fmt, "persistence.txt")
step("Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB)",
[PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE],
"persistence.txt", append=True)
step("Persistence: контейнеры после (Status = Up N seconds)", dc + fmt, "persistence.txt", append=True)
step("Persistence: сверка БД <-> S3 после рестарта", [PY, "scripts/ops.py", "reconcile"], "persistence.txt",
append=True)
if not args.quick:
step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30",
"--concurrency", "8", "--output", "evidence/load.json"], "load.txt")
step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format",
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True)
step("Лог LB (последние строки, без секретов)", dc + ["logs", "--no-color", "--tail", "60", "lb"], "lb_log_sample.txt")
summary = ["# %s" % stamp()] + ["%-4s %-70s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS]
(EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8")
print("\n" + "\n".join(summary))
sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1)
if __name__ == "__main__":
main()