forked from mtusi2026/mtusi2026lab5
231 lines
15 KiB
Python
231 lines
15 KiB
Python
#!/usr/bin/env python3
|
||
"""Прогон всех проверок lab5 (и унаследованных lab2-4) на запущенном compose-стенде с записью в evidence/.
|
||
|
||
python scripts/collect_evidence.py # ~18-22 минуты
|
||
python scripts/collect_evidence.py --quick # ~10-12 минут: без kill-серии, rolling update и нагрузки
|
||
|
||
Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`,
|
||
`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики,
|
||
перезапускает хранилища и S3 (volumes сохраняются), останавливает S3 и делает
|
||
rolling update lab5 -> lab5.1 -> lab5 (откат), останавливает worker/broker, поэтому запускайте его только на учебном стенде.
|
||
Токены и пароли в файлы не пишутся (тесты держат их в памяти).
|
||
"""
|
||
import argparse
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
from urllib.request import Request, urlopen
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
EV = ROOT / "evidence"
|
||
PY = sys.executable or "python"
|
||
WIN = os.name == "nt"
|
||
HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh"))
|
||
HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh"))
|
||
BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080")
|
||
RESULTS = []
|
||
|
||
try:
|
||
sys.stdout.reconfigure(errors="replace")
|
||
except AttributeError:
|
||
pass
|
||
|
||
|
||
def stamp():
|
||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||
|
||
|
||
def step(title, cmd, out_file, append=False, env=None, timeout=900):
|
||
"""Выполнить команду, сохранить «$ команда + вывод» в evidence/<out_file>."""
|
||
print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True)
|
||
started = time.monotonic()
|
||
try:
|
||
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||
timeout=timeout, env=dict(os.environ, **(env or {})),
|
||
encoding="utf-8", errors="replace")
|
||
output, rc = proc.stdout, proc.returncode
|
||
except subprocess.TimeoutExpired as exc:
|
||
output, rc = (exc.stdout or "") + "\nTIMEOUT", 124
|
||
elapsed = time.monotonic() - started
|
||
shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "")
|
||
text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip())
|
||
path = EV / out_file
|
||
with path.open("a" if append else "w", encoding="utf-8") as f:
|
||
f.write(text + ("\n" if append else ""))
|
||
print(output.rstrip()[-1500:], flush=True)
|
||
RESULTS.append((title, rc, out_file))
|
||
return rc, output
|
||
|
||
|
||
def instances_through_lb(n=20):
|
||
"""Одна сессия, n запросов через LB -> какие реплики отвечали."""
|
||
import base64
|
||
import uuid
|
||
name = "inst_" + uuid.uuid4().hex[:12]
|
||
pw = "Inst-" + uuid.uuid4().hex
|
||
body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode()
|
||
urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10)
|
||
cred = base64.b64encode((name + ":" + pw).encode()).decode()
|
||
sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST",
|
||
headers={"Authorization": "Basic " + cred}), timeout=10).read())
|
||
lines = []
|
||
for i in range(n):
|
||
with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r:
|
||
lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % (
|
||
i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id")))
|
||
return lines
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--quick", action="store_true", help="без SIGKILL-серии, rolling update и нагрузки")
|
||
args = ap.parse_args()
|
||
EV.mkdir(exist_ok=True)
|
||
(EV / "ops.log").write_text("", encoding="utf-8")
|
||
dc = ["docker", "compose", "--env-file", ".env"]
|
||
|
||
step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt")
|
||
step("Compose config", dc + ["config", "--quiet"], "check.txt", append=True)
|
||
step("Стенд: контейнеры", dc + ["ps", "-a"], "stand.txt")
|
||
step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True)
|
||
step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format",
|
||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True)
|
||
step("Стенд: опубликованные порты (только lb)", dc + ["ps", "--format", "table {{.Service}}\t{{.Ports}}"],
|
||
"stand.txt", append=True)
|
||
step("Образ SeaweedFS (закреплённая версия)", dc + ["images", "s3"], "stand.txt", append=True)
|
||
step("lab5: broker (Redis AOF) и worker", dc + ["ps", "broker", "worker"], "stand.txt", append=True)
|
||
step("lab5: конфигурация durability брокера", dc + ["exec", "-T", "broker", "sh", "-c",
|
||
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET appendfsync; "
|
||
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET maxmemory-policy"],
|
||
"stand.txt", append=True)
|
||
|
||
# Bucket bootstrap: первый запуск был при up; повтор — no-op.
|
||
step("S3 bootstrap: лог первого запуска s3-init", dc + ["logs", "--no-color", "s3-init"], "s3_bootstrap.txt")
|
||
step("S3 bootstrap: повторный запуск (идемпотентность)", dc + ["run", "--rm", "-T", "s3-init"],
|
||
"s3_bootstrap.txt", append=True)
|
||
|
||
lines = instances_through_lb()
|
||
(EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n"
|
||
% (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})),
|
||
encoding="utf-8")
|
||
print("\n".join(lines))
|
||
|
||
step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt")
|
||
step("Собственные тесты (HTTP через LB, включая test_attachments.py)",
|
||
[PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt")
|
||
step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"],
|
||
"student-tests.txt", append=True)
|
||
step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"],
|
||
"student-tests.txt", append=True)
|
||
step("Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*)",
|
||
[PY, "scripts/ops.py", "in-network", "consistency"], "consistency.txt")
|
||
step("Сверка БД <-> S3 (dry-run)", [PY, "scripts/ops.py", "reconcile"], "reconcile.txt")
|
||
step("Приватный бакет вне API, права ключей, закрытые админ-порты", [PY, "scripts/ops.py", "s3-check"],
|
||
"s3_private.txt")
|
||
step("На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes",
|
||
[PY, "scripts/ops.py", "api-fs"], "s3_private.txt", append=True)
|
||
step("API: файлы в /tmp (tmpfs) после загрузок",
|
||
dc + ["exec", "-T", "api", "sh", "-c", "df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l)"],
|
||
"s3_private.txt", append=True)
|
||
|
||
step("Пик памяти: 10 одновременных загрузок по 9.5 MiB",
|
||
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "10", "--rounds", "3",
|
||
"--docker", "--label", "c=10, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c10.json"],
|
||
"upload_memory.txt")
|
||
step("Пик памяти: 20 одновременных загрузок",
|
||
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "20", "--rounds", "2",
|
||
"--docker", "--label", "c=20, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c20.json"],
|
||
"upload_memory.txt", append=True)
|
||
|
||
# Публичный тест останавливает реплику, обслужившую его первое сообщение.
|
||
# Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь
|
||
# лишними запросами, пока цель не окажется другой (README: «повторить для второй»).
|
||
targets = []
|
||
for i in range(1, 5):
|
||
if len(set(targets)) >= 2:
|
||
break
|
||
for _ in range(i - 1):
|
||
urlopen(BASE + "/health/live", timeout=5).read()
|
||
ops_log = EV / "ops.log"
|
||
stops_before = sum(1 for l in ops_log.read_text(encoding="utf-8").splitlines()
|
||
if "stop-one: mode=" in l) if ops_log.exists() else 0
|
||
step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1)
|
||
step("Failover #%d: make test-failover (вкл. скачивание файла через другую реплику)" % i,
|
||
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE],
|
||
"failover.txt", append=True)
|
||
step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"],
|
||
"failover.txt", append=True)
|
||
step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True)
|
||
stops = [l for l in ops_log.read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l] \
|
||
if ops_log.exists() else []
|
||
# Только остановка из ЭТОЙ попытки: если тест упал до hook'а, новой строки нет,
|
||
# и цель прошлой попытки не должна засчитываться повторно.
|
||
if len(stops) > stops_before:
|
||
targets.append(stops[-1].split("target=")[1].split()[0])
|
||
with (EV / "failover.txt").open("a", encoding="utf-8") as f:
|
||
f.write("# остановленные публичным тестом реплики: %s\n" % targets)
|
||
|
||
step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE,
|
||
"--output", "evidence/failover_series.json"], "failover_series.txt")
|
||
if not args.quick:
|
||
step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE,
|
||
"--mode", "kill", "--output", "evidence/failover_series_kill.json"],
|
||
"failover_series.txt", append=True)
|
||
|
||
step("Недоступность S3 и восстановление", [PY, "student-tests/s3_outage.py", "--base-url", BASE,
|
||
"--output", "evidence/s3_outage.json"], "s3_outage.txt")
|
||
|
||
# lab5: очередь и worker
|
||
step("Варианты изображений по фактическим байтам (test_pipeline.py)",
|
||
[PY, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_pipeline.py", "-v"],
|
||
"pipeline.txt", env={"BASE_URL": BASE})
|
||
step("Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready",
|
||
[PY, "student-tests/worker_outage.py", "--base-url", BASE, "--output", "evidence/worker_outage.json"],
|
||
"worker_outage.txt")
|
||
step("Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)",
|
||
[PY, "scripts/ops.py", "pipeline-faults"], "pipeline_faults.txt", timeout=1200)
|
||
step("Очередь после сценариев: задания, outbox, DLQ, lag", [PY, "scripts/ops.py", "jobs", "stats"],
|
||
"pipeline_faults.txt", append=True)
|
||
step("Реестр failed (DLQ)", [PY, "scripts/ops.py", "jobs", "failed", "--limit", "10"], "pipeline_faults.txt",
|
||
append=True)
|
||
step("Worker: последние строки лога", dc + ["logs", "--no-color", "--tail", "80", "worker"], "worker_log_sample.txt")
|
||
|
||
if not args.quick:
|
||
step("Rolling update lab5 -> lab5.1 под нагрузкой",
|
||
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"],
|
||
"rolling_update.txt", env={"APP_VERSION": "lab5.1"})
|
||
step("Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой",
|
||
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"],
|
||
"rolling_update.txt", append=True, env={"APP_VERSION": "lab5"})
|
||
step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True)
|
||
|
||
fmt = ["ps", "-a", "--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"]
|
||
step("Persistence: контейнеры до", dc + fmt, "persistence.txt")
|
||
step("Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB)",
|
||
[PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE],
|
||
"persistence.txt", append=True)
|
||
step("Persistence: контейнеры после (Status = Up N seconds)", dc + fmt, "persistence.txt", append=True)
|
||
step("Persistence: сверка БД <-> S3 после рестарта", [PY, "scripts/ops.py", "reconcile"], "persistence.txt",
|
||
append=True)
|
||
|
||
if not args.quick:
|
||
step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30",
|
||
"--concurrency", "8", "--output", "evidence/load.json"], "load.txt")
|
||
step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format",
|
||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True)
|
||
|
||
step("Лог LB (последние строки, без секретов)", dc + ["logs", "--no-color", "--tail", "60", "lb"], "lb_log_sample.txt")
|
||
|
||
summary = ["# %s" % stamp()] + ["%-4s %-70s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS]
|
||
(EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8")
|
||
print("\n" + "\n".join(summary))
|
||
sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|