forked from mtusi2026/mtusi2026lab3
136 lines
5.1 KiB
Python
136 lines
5.1 KiB
Python
"""Полная проверка lab3. Использует текущий учебный стенд, volumes сохраняет."""
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
from benchmark import Client, fixture, measure
|
|
from replicas import compose, containers, describe, docker, instance_id, wait_ready, wait_visible
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
EVIDENCE = ROOT / "evidence"
|
|
|
|
|
|
def run_test(name, *args, stdin=None):
|
|
result = subprocess.run(args, cwd=ROOT, input=stdin, text=True, capture_output=True)
|
|
(EVIDENCE / name).write_text(result.stdout + result.stderr)
|
|
print(name, "PASS" if result.returncode == 0 else "FAIL", flush=True)
|
|
if result.returncode:
|
|
raise RuntimeError((result.stdout + result.stderr)[-3000:])
|
|
|
|
|
|
def main():
|
|
py = sys.executable
|
|
for filename in ["smoke", "lab3", "extra", "regressions", "production"]:
|
|
run_test("lab3-" + filename + ".txt", py, "tests/" + filename + ".py")
|
|
run_test(
|
|
"lab3-db-idempotency.txt",
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
containers()[0],
|
|
"python",
|
|
stdin=(ROOT / "tests/idempotency_db.py").read_text(),
|
|
)
|
|
run_test(
|
|
"lab3-public-failover.txt",
|
|
py,
|
|
"tests/resilience.py",
|
|
"--mode",
|
|
"failover",
|
|
"--action-script",
|
|
"scripts/stop-one.sh",
|
|
)
|
|
compose("up", "-d", "--no-deps", "--wait", "api")
|
|
for c in containers():
|
|
wait_visible(wait_ready(c))
|
|
events = []
|
|
actor, chat = fixture()
|
|
initial = containers()
|
|
for target in initial:
|
|
alias = instance_id(target)
|
|
print("Failure test", target[:12], alias, flush=True)
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
event = []
|
|
load = pool.submit(measure, actor, chat, 12, 8, event)
|
|
time.sleep(3)
|
|
event.append(time.monotonic())
|
|
docker("kill", "--signal", "KILL", target)
|
|
report, confirmations = load.result()
|
|
assert not describe(target)["Running"]
|
|
assert report["successes"].get("read") and report["successes"].get("write")
|
|
assert all(report["seconds"][b]["success"] > 0 for b in range(5, 11)), (
|
|
"No service after detection"
|
|
)
|
|
c = Client()
|
|
# Each acknowledged message is read back with the same idempotency key.
|
|
for key, mid in confirmations:
|
|
response = c.message(actor, chat, "load-" + key, key=key)
|
|
assert response["json"]["id"] == mid
|
|
assert response["headers"]["X-Instance-Id"] != alias
|
|
report.update(
|
|
{
|
|
"stopped_container": target[:12],
|
|
"stopped_instance": alias,
|
|
"exit_code": describe(target)["ExitCode"],
|
|
"all_acknowledged_messages_replayed": len(confirmations),
|
|
"failure_started_at_load_second": 3,
|
|
"service_after_second_5": "reads/writes continue",
|
|
"note": "hard SIGKILL, no automatic retries in load, replay confirmations after failure",
|
|
}
|
|
)
|
|
events.append(report)
|
|
docker("start", target)
|
|
wait_visible(wait_ready(target))
|
|
(EVIDENCE / "lab3-failures.json").write_text(
|
|
json.dumps(events, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print("Both replicas survived sequential hard-failure tests", flush=True)
|
|
actor, chat = fixture()
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
load = pool.submit(measure, actor, chat, 40, 8)
|
|
time.sleep(3)
|
|
run_test("lab3-rolling.txt", py, "scripts/rolling-update.py")
|
|
report, confirmations = load.result()
|
|
(EVIDENCE / "lab3-rolling-load.json").write_text(json.dumps(report, indent=2) + "\n")
|
|
assert report["successes"].get("read") and report["successes"].get("write")
|
|
# Zero errors are not required by rubric; any transitional errors are reported.
|
|
client = Client()
|
|
for key, mid in confirmations:
|
|
assert client.message(actor, chat, "load-" + key, key=key)["json"]["id"] == mid
|
|
before = {c: describe(c)["StartedAt"] for c in containers()}
|
|
storage_before = {
|
|
s: describe(compose("ps", "-q", s))["StartedAt"] for s in ["postgres", "redis"]
|
|
}
|
|
run_test(
|
|
"lab3-persistence.txt",
|
|
py,
|
|
"tests/resilience.py",
|
|
"--mode",
|
|
"persistence",
|
|
"--action-script",
|
|
"scripts/restart.sh",
|
|
)
|
|
after = {c: describe(c)["StartedAt"] for c in containers()}
|
|
storage_after = {
|
|
s: describe(compose("ps", "-q", s))["StartedAt"] for s in ["postgres", "redis"]
|
|
}
|
|
assert set(before) != set(after) or before != after
|
|
assert all(storage_before[s] != storage_after[s] for s in storage_before)
|
|
proof = {
|
|
"api_before": before,
|
|
"api_after": after,
|
|
"storage_before": storage_before,
|
|
"storage_after": storage_after,
|
|
}
|
|
(EVIDENCE / "lab3-restart-proof.json").write_text(json.dumps(proof, indent=2) + "\n")
|
|
run_test("lab3-final-smoke.txt", py, "tests/smoke.py")
|
|
print("PASS: all lab3 acceptance checks", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|