forked from mtusi2026/mtusi2026lab3
+535
@@ -0,0 +1,535 @@
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from urllib.error import URLError
|
||||
from urllib.request import ProxyHandler, build_opener
|
||||
import venv
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
LOG = []
|
||||
LOCAL_HTTP = build_opener(ProxyHandler({}))
|
||||
|
||||
|
||||
def settings():
|
||||
path = ROOT / ".env"
|
||||
result = {}
|
||||
if path.exists():
|
||||
for line in path.read_text(encoding="utf-8-sig").splitlines():
|
||||
key, separator, value = line.partition("=")
|
||||
if separator and not key.strip().startswith("#"):
|
||||
result[key.strip()] = value.strip()
|
||||
return result
|
||||
|
||||
|
||||
def environment(args):
|
||||
result = os.environ.copy()
|
||||
result.update(PYTHONUTF8="1", PYTHONIOENCODING="utf-8", PYTHONDONTWRITEBYTECODE="1",
|
||||
COMPOSE_PROJECT_NAME=args.project, HTTP_PORT=str(args.port),
|
||||
SESSION_TTL_SECONDS=str(args.ttl), LAB3_PROJECT=args.project,
|
||||
LAB3_PORT=str(args.port), LAB3_TTL=str(args.ttl))
|
||||
for key in ("NO_PROXY", "no_proxy"):
|
||||
result[key] = ",".join(filter(None, [result.get(key), "localhost", "127.0.0.1"]))
|
||||
return result
|
||||
|
||||
|
||||
def say(value):
|
||||
line = str(value).replace(str(ROOT), ".")
|
||||
print(line, flush=True)
|
||||
LOG.append(line)
|
||||
|
||||
|
||||
def run(args, command, quiet=False):
|
||||
command = [str(item) for item in command]
|
||||
if not quiet:
|
||||
say("$ " + subprocess.list2cmdline(command))
|
||||
with subprocess.Popen(command, cwd=ROOT, env=environment(args), stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace") as process:
|
||||
lines = []
|
||||
for line in process.stdout:
|
||||
lines.append(line)
|
||||
if not quiet:
|
||||
say(line.rstrip())
|
||||
code = process.wait()
|
||||
if code:
|
||||
raise RuntimeError(f"{command[0]} exited with code {code}")
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def compose(args, *command, quiet=False):
|
||||
if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"):
|
||||
raise RuntimeError("This runner only manages Kuznetsov lab3 projects")
|
||||
return run(args, ["docker", "compose", "--project-name", args.project, "--env-file", ".env", *command], quiet)
|
||||
|
||||
|
||||
def interpreter():
|
||||
return ROOT / ".venv" / ("Scripts/python.exe" if os.name == "nt" else "bin/python")
|
||||
|
||||
|
||||
def evidence(name, value):
|
||||
path = ROOT / "evidence" / name
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
content = json.dumps(value, ensure_ascii=False, indent=2) if not isinstance(value, str) else value
|
||||
path.write_text(content.rstrip() + "\n", encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def setup(args):
|
||||
if not (ROOT / ".env").exists():
|
||||
shutil.copyfile(ROOT / ".env.example", ROOT / ".env")
|
||||
folder = ROOT / "secrets"
|
||||
folder.mkdir(exist_ok=True)
|
||||
password_file = folder / "postgres_password"
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(32) + "\n", encoding="ascii")
|
||||
password = password_file.read_text(encoding="ascii").strip()
|
||||
values = {"database_url": f"postgresql://messenger:{password}@db:5432/messenger",
|
||||
"cursor_key": secrets.token_hex(32)}
|
||||
for name, value in values.items():
|
||||
path = folder / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n", encoding="ascii")
|
||||
if os.name != "nt":
|
||||
path.chmod(0o644)
|
||||
if os.name != "nt":
|
||||
folder.chmod(0o700)
|
||||
password_file.chmod(0o644)
|
||||
if not interpreter().exists():
|
||||
venv.EnvBuilder(with_pip=True).create(ROOT / ".venv")
|
||||
lockfiles = [ROOT / "requirements.lock", ROOT / "requirements-validation.lock"]
|
||||
fingerprint = hashlib.sha256(b"".join(path.read_bytes() for path in lockfiles)).hexdigest()
|
||||
marker = ROOT / ".venv/.lab-dependencies"
|
||||
if not marker.exists() or marker.read_text(encoding="ascii") != fingerprint:
|
||||
run(args, [interpreter(), "-X", "utf8", "-m", "pip", "install", "--disable-pip-version-check",
|
||||
"--no-input", "--progress-bar", "off", "--quiet", "-r", lockfiles[0], "-r", lockfiles[1]])
|
||||
marker.write_text(fingerprint, encoding="ascii")
|
||||
say("Environment ready; existing settings and secrets preserved")
|
||||
|
||||
|
||||
def base_url(args):
|
||||
return f"http://127.0.0.1:{args.port}"
|
||||
|
||||
|
||||
def wait_ready(args, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with LOCAL_HTTP.open(base_url(args) + "/health/ready", timeout=4) as response:
|
||||
if response.status == 200 and json.load(response) == {"status": "ok"}:
|
||||
return
|
||||
except (URLError, OSError, ValueError):
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError("API readiness timeout")
|
||||
|
||||
|
||||
def build(args):
|
||||
compose(args, "config", "--quiet")
|
||||
compose(args, "build", "api1")
|
||||
compose(args, "run", "--rm", "--no-deps", "lb", "haproxy", "-c", "-f",
|
||||
"/usr/local/etc/haproxy/haproxy.cfg")
|
||||
|
||||
|
||||
def start(args):
|
||||
current = compose(args, "ps", "--all", "--quiet", "lb", quiet=True).strip()
|
||||
if not current:
|
||||
with socket.socket() as probe:
|
||||
try:
|
||||
probe.bind(("127.0.0.1", args.port))
|
||||
except OSError:
|
||||
raise RuntimeError(f"Port {args.port} is busy; stop its container or use --port") from None
|
||||
compose(args, "up", "-d", "--no-build", "db", "migrate", "api1", "api2", "lb")
|
||||
wait_ready(args)
|
||||
wait_replicas(args)
|
||||
say("Two API replicas ready through LB: " + base_url(args))
|
||||
|
||||
|
||||
def states(args):
|
||||
result = {}
|
||||
for service in ("api1", "api2", "db", "lb"):
|
||||
container = compose(args, "ps", "--all", "--quiet", service, quiet=True).strip()
|
||||
if not container:
|
||||
raise RuntimeError("Missing container: " + service)
|
||||
data = json.loads(run(args, ["docker", "inspect", container], quiet=True))[0]
|
||||
labels = data["Config"].get("Labels", {})
|
||||
if labels.get("com.docker.compose.project") != args.project or labels.get("com.docker.compose.service") != service:
|
||||
raise RuntimeError("Unexpected container ownership")
|
||||
result[service] = {
|
||||
"id": data["Id"], "image": data["Image"], "started_at": data["State"]["StartedAt"],
|
||||
"running": data["State"]["Running"], "user": data["Config"]["User"],
|
||||
"volumes": [mount["Name"] for mount in data["Mounts"] if mount["Type"] == "volume"],
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def restart(args):
|
||||
before = states(args)
|
||||
if not all(item["running"] for item in before.values()):
|
||||
raise RuntimeError("All runtime containers must be running before restart")
|
||||
compose(args, "restart", "--timeout", "10", "db", "api1", "api2", "lb")
|
||||
after = states(args)
|
||||
for service in before:
|
||||
if before[service]["started_at"] == after[service]["started_at"]:
|
||||
raise RuntimeError("Container was not restarted: " + service)
|
||||
if before[service]["volumes"] != after[service]["volumes"]:
|
||||
raise RuntimeError("Volume changed during restart")
|
||||
evidence("restart.json", {"project": args.project, "before": before, "after": after})
|
||||
say("PASS: both APIs, PostgreSQL and LB restarted, volume preserved")
|
||||
|
||||
|
||||
def contract_tests(args):
|
||||
run(args, [interpreter(), "-X", "utf8", "scripts/check_contract.py"])
|
||||
run(args, [interpreter(), "-X", "utf8", "scripts/validate_openapi.py"])
|
||||
compose(args, "config", "--quiet")
|
||||
|
||||
|
||||
def api_tests(args):
|
||||
wait_ready(args)
|
||||
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
|
||||
os.environ["BASE_URL"] = base_url(args)
|
||||
run(args, [interpreter(), "-X", "utf8", "-m", "unittest", "discover", "-s", "student-tests", "-v"])
|
||||
|
||||
|
||||
def client(args):
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
for key in ("NO_PROXY", "no_proxy"):
|
||||
os.environ[key] = environment(args)[key]
|
||||
return Client(base_url(args))
|
||||
|
||||
|
||||
def database_stats(args):
|
||||
sql = """SELECT json_build_object(
|
||||
'users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),
|
||||
'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),
|
||||
'sessions',(SELECT count(*) FROM sessions),'migrations',(SELECT count(*) FROM schema_migrations),
|
||||
'idempotency_keys',(SELECT count(*) FROM idempotency_keys),
|
||||
'expiry_fingerprint',(SELECT md5(COALESCE(string_agg(user_id::text || created_at::text || expires_at::text,
|
||||
',' ORDER BY token_hash),'')) FROM sessions),
|
||||
'passwords_are_argon2id',(SELECT bool_and(password_hash LIKE '$argon2id$%') FROM users),
|
||||
'session_hashes_only',(SELECT bool_and(token_hash ~ '^[0-9a-f]{64}$') FROM sessions))"""
|
||||
result = compose(args, "exec", "-T", "db", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", sql, quiet=True)
|
||||
return json.loads(result)
|
||||
|
||||
|
||||
def restart_check(args):
|
||||
from urllib.parse import urlencode
|
||||
c = client(args)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
replay_key = secrets.token_hex(16)
|
||||
original = c.message(alice, chat, "first", key=replay_key)["json"]
|
||||
last = c.message(bob, chat, "second")["json"]
|
||||
page = c.call("GET", path + "?limit=1", actor=alice, schema="MessagePage")["json"]
|
||||
revoked = c.user("revoked")
|
||||
c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=revoked)
|
||||
hook = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
|
||||
run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", hook, "--base-url", base_url(args)])
|
||||
continued = c.call("GET", path + "?" + urlencode({"cursor": page["next_cursor"]}),
|
||||
actor=alice, schema="MessagePage")["json"]
|
||||
if continued != {"items": [last], "next_cursor": None}:
|
||||
raise RuntimeError("Old cursor did not survive restart")
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=revoked)
|
||||
c.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)
|
||||
before = database_stats(args)
|
||||
compose(args, "run", "--rm", "--no-deps", "migrate")
|
||||
compose(args, "run", "--rm", "--no-deps", "migrate")
|
||||
after = database_stats(args)
|
||||
if before != after or after["migrations"] != 2:
|
||||
raise RuntimeError("Migration replay changed existing data")
|
||||
if not after["passwords_are_argon2id"] or not after["session_hashes_only"]:
|
||||
raise RuntimeError("Unexpected credential representation in DB")
|
||||
repeated = c.message(alice, chat, "first", key=replay_key)["json"]
|
||||
if repeated != original:
|
||||
raise RuntimeError("Idempotency result was not preserved")
|
||||
result = {"idempotency_after_restart": "PASS", "public_persistence": "PASS", "cursor_after_restart": "PASS",
|
||||
"logout_after_restart": "PASS", "migration_replay": "PASS", "database": after}
|
||||
evidence("persistence.json", result)
|
||||
say("PASS: old session/cursor, durable logout and repeatable migrations")
|
||||
return result
|
||||
|
||||
|
||||
def ttl_check(args):
|
||||
test = argparse.Namespace(project=args.project + "-ttl", port=args.test_port, ttl=2,
|
||||
test_port=args.test_port, action="start")
|
||||
try:
|
||||
start(test)
|
||||
c = client(test)
|
||||
actor = c.user("expiry")
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
before = database_stats(test)["expiry_fingerprint"]
|
||||
compose(test, "restart", "--timeout", "3", "api1", "api2")
|
||||
wait_ready(test)
|
||||
after = database_stats(test)["expiry_fingerprint"]
|
||||
if before != after:
|
||||
raise RuntimeError("Session expiry changed during restart")
|
||||
time.sleep(2.2)
|
||||
c.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
registration = actor["registration"]
|
||||
import base64
|
||||
basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode()
|
||||
c.call("POST", "/api/v1/auth/sessions", 201, headers={"Authorization": "Basic " + basic}, schema="Session")
|
||||
say("PASS: TTL=2, expired token rejected after API restart; fresh login works")
|
||||
result = {"ttl_seconds": 2, "expiry_unchanged_on_restart": True, "before_restart": 200,
|
||||
"after_expiry_and_restart": 401, "new_login": 201}
|
||||
evidence("expiry.json", result)
|
||||
return result
|
||||
finally:
|
||||
compose(test, "down", "--timeout", "10")
|
||||
|
||||
|
||||
def dependency_check(args):
|
||||
c = client(args)
|
||||
actor = c.user("dbcheck")
|
||||
try:
|
||||
compose(args, "stop", "--timeout", "10", "db")
|
||||
c.call("GET", "/health/live", schema="Health")
|
||||
outage_response(args, "/health/ready")
|
||||
outage_response(args, "/api/v1/users/me", actor["token"])
|
||||
finally:
|
||||
compose(args, "start", "db")
|
||||
wait_ready(args)
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
say("PASS: DB outage -> live=200, ready/business=503; session works after recovery")
|
||||
return {"live_without_db": 200, "ready_without_db": 503, "business_without_db": 503, "recovered": 200}
|
||||
|
||||
|
||||
def verify(args):
|
||||
summary = {"date_utc": datetime.now(timezone.utc).isoformat(), "platform": platform.platform(),
|
||||
"source_commit": run(args, ["git", "rev-parse", "HEAD"], quiet=True).strip(),
|
||||
"worktree_dirty_at_start": bool(run(args, ["git", "status", "--porcelain"], quiet=True).strip()),
|
||||
"project": args.project, "status": "FAIL"}
|
||||
try:
|
||||
setup(args)
|
||||
contract_tests(args)
|
||||
build(args)
|
||||
start(args)
|
||||
api_tests(args)
|
||||
summary["persistence"] = restart_check(args)
|
||||
summary["failover"] = failover_check(args)
|
||||
summary["expiry"] = ttl_check(args)
|
||||
summary["dependency"] = dependency_check(args)
|
||||
run(args, [interpreter(), "-X", "utf8", "tests/smoke.py", "--base-url", base_url(args)])
|
||||
summary["containers"] = states(args)
|
||||
summary["docker"] = run(args, ["docker", "version", "--format", "{{.Server.Version}}"], quiet=True).strip()
|
||||
summary["resources"] = json.loads(run(
|
||||
args, ["docker", "info", "--format", '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'], quiet=True))
|
||||
summary["status"] = "PASS"
|
||||
say("PASS: full lab3 verification")
|
||||
finally:
|
||||
evidence("verification.txt", "\n".join(LOG))
|
||||
evidence("summary.json", summary)
|
||||
|
||||
|
||||
def instance_map(args):
|
||||
current = states(args)
|
||||
result = {}
|
||||
for service in ("api1", "api2"):
|
||||
item = current[service]
|
||||
if not item["running"]:
|
||||
continue
|
||||
code = (
|
||||
"import urllib.request; "
|
||||
"o=urllib.request.build_opener(urllib.request.ProxyHandler({})); "
|
||||
"r=o.open('http://127.0.0.1:8080/health/live',timeout=1); "
|
||||
"print(r.headers['X-Instance-Id'])"
|
||||
)
|
||||
alias = run(args, ["docker", "exec", item["id"], "python", "-c", code], quiet=True).strip()
|
||||
import re
|
||||
if not re.fullmatch(r"[0-9a-f]{32}", alias):
|
||||
raise RuntimeError("Invalid API instance alias")
|
||||
result[service] = {"instance_id": alias, "container_id": item["id"]}
|
||||
return result
|
||||
|
||||
|
||||
def wait_replicas(args):
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
mapping = instance_map(args)
|
||||
if len(mapping) == 2:
|
||||
c = client(args)
|
||||
seen = {c.call("GET", "/health/ready", schema="Health")["headers"]["X-Instance-Id"]
|
||||
for _ in range(8)}
|
||||
if seen == {item["instance_id"] for item in mapping.values()}:
|
||||
return mapping
|
||||
except (RuntimeError, OSError, AssertionError):
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError("Two healthy API replicas were not observed through LB")
|
||||
|
||||
|
||||
def restore(args):
|
||||
compose(args, "start", "api1", "api2")
|
||||
wait_ready(args)
|
||||
return wait_replicas(args)
|
||||
|
||||
|
||||
def stop_one(args):
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "")
|
||||
before = instance_map(args)
|
||||
matches = [(name, item) for name, item in before.items() if item["instance_id"] == target]
|
||||
if len(before) != 2 or len(matches) != 1:
|
||||
raise RuntimeError("Target must match exactly one of two running API replicas")
|
||||
service, item = matches[0]
|
||||
started = time.monotonic()
|
||||
run(args, ["docker", "stop", "--timeout", "3", item["container_id"]])
|
||||
after = states(args)
|
||||
if after[service]["running"] or not after["api1" if service == "api2" else "api2"]["running"]:
|
||||
raise RuntimeError("Unexpected API state after stop")
|
||||
elapsed = time.monotonic() - started
|
||||
if elapsed >= 10:
|
||||
raise RuntimeError("Stopping the selected API exceeded 10 seconds")
|
||||
evidence("stop-one.json", {"service": service, "target_instance": target, "before": before,
|
||||
"after": after, "stop_seconds": round(elapsed, 3)})
|
||||
say("Stopped only " + service)
|
||||
|
||||
|
||||
def failover_check(args):
|
||||
wait_replicas(args)
|
||||
hook = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
||||
try:
|
||||
output = run(args, [interpreter(), "-X", "utf8", "tests/resilience.py", "--mode", "failover",
|
||||
"--action-script", hook, "--base-url", base_url(args)])
|
||||
first = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
|
||||
first["metrics"] = next(json.loads(line) for line in output.splitlines()
|
||||
if line.startswith("{") and "first_recovery_seconds" in line)
|
||||
if first["after"][first["service"]]["running"]:
|
||||
raise RuntimeError("Public hook did not leave the target stopped")
|
||||
finally:
|
||||
restore(args)
|
||||
other_service = "api2" if first["service"] == "api1" else "api1"
|
||||
mapping = instance_map(args)
|
||||
target = mapping[other_service]["instance_id"]
|
||||
c = client(args)
|
||||
c.timeout = 2
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
for _ in range(10):
|
||||
replay_key = secrets.token_hex(16)
|
||||
replay_text = "before-failure-" + replay_key
|
||||
response = c.message(alice, chat, replay_text, key=replay_key)
|
||||
if response["headers"]["X-Instance-Id"] == target:
|
||||
original = response["json"]
|
||||
break
|
||||
else:
|
||||
raise RuntimeError("Second API was not reached through LB")
|
||||
env = environment(args)
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
started = time.monotonic()
|
||||
process = subprocess.Popen([str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
from client import Failure
|
||||
from resilience import history
|
||||
confirmations, after_instances = [], []
|
||||
failures = 0
|
||||
recovered = None
|
||||
key = secrets.token_hex(16)
|
||||
text = "during-failure-" + key
|
||||
try:
|
||||
while time.monotonic() - started < 15:
|
||||
code = process.poll()
|
||||
if code not in (None, 0):
|
||||
raise RuntimeError("Second stop hook failed")
|
||||
if code is None and time.monotonic() - started >= 10:
|
||||
raise RuntimeError("Second stop hook exceeded 10 seconds")
|
||||
try:
|
||||
response = c.message(alice, chat, text, key=key)
|
||||
confirmations.append(response["json"])
|
||||
if code == 0:
|
||||
if recovered is None:
|
||||
recovered = time.monotonic() - started
|
||||
after_instances.append(response["headers"]["X-Instance-Id"])
|
||||
key = secrets.token_hex(16)
|
||||
text = "during-failure-" + key
|
||||
except Failure:
|
||||
failures += 1
|
||||
time.sleep(0.1)
|
||||
if process.poll() != 0 or recovered is None or recovered > 10:
|
||||
raise RuntimeError("Second API failure did not recover within 10 seconds")
|
||||
if len(after_instances) < 5 or target in after_instances[-5:]:
|
||||
raise RuntimeError("Stopped API still serves requests")
|
||||
replayed = c.message(alice, chat, replay_text, key=replay_key)
|
||||
if replayed["json"] != original or replayed["headers"]["X-Instance-Id"] == target:
|
||||
raise RuntimeError("Message replay through the survivor changed the result")
|
||||
messages = history(c, bob, chat)
|
||||
ids = [item["id"] for item in messages]
|
||||
texts = [item["text"] for item in messages]
|
||||
if len(ids) != len(set(ids)) or len(texts) != len(set(texts)):
|
||||
raise RuntimeError("Duplicates appeared during failover")
|
||||
if any(item not in messages for item in [original, *confirmations]):
|
||||
raise RuntimeError("A confirmed message was lost")
|
||||
second = json.loads((ROOT / "evidence/stop-one.json").read_text(encoding="utf-8"))
|
||||
if second["service"] != other_service:
|
||||
raise RuntimeError("Wrong second API was stopped")
|
||||
second["metrics"] = {"successful_writes": len(confirmations), "failed_attempts": failures,
|
||||
"first_recovery_seconds": round(recovered, 3),
|
||||
"replay_on_survivor": "PASS"}
|
||||
result = {"public": first, "second_replica": second, "status": "PASS"}
|
||||
evidence("failover.json", result)
|
||||
say("PASS: both physical API replicas stopped separately; no lost messages or duplicates")
|
||||
return result
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.terminate()
|
||||
process.wait(timeout=5)
|
||||
restore(args)
|
||||
|
||||
|
||||
def outage_response(args, path, token=None):
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request
|
||||
from client import shape
|
||||
headers = {"Authorization": "Bearer " + token} if token else {}
|
||||
try:
|
||||
response = LOCAL_HTTP.open(Request(base_url(args) + path, headers=headers), timeout=10)
|
||||
except HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
payload = json.load(response)
|
||||
if response.status != 503 or not response.headers.get("X-Request-Id"):
|
||||
raise RuntimeError("Unexpected dependency failure response")
|
||||
shape(payload, "Error")
|
||||
if payload["error"]["request_id"] != response.headers["X-Request-Id"]:
|
||||
raise RuntimeError("LB error request ID mismatch")
|
||||
|
||||
|
||||
def main():
|
||||
saved = settings()
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("action", choices=["setup", "build", "start", "test", "restart", "stop-one",
|
||||
"restart-check", "failover-check", "restore", "stop", "verify"])
|
||||
parser.add_argument("--project", default=os.environ.get("LAB3_PROJECT", saved.get("COMPOSE_PROJECT_NAME", "kuznetsov-lab3")))
|
||||
parser.add_argument("--port", type=int, default=int(os.environ.get("LAB3_PORT", saved.get("HTTP_PORT", "18103"))))
|
||||
parser.add_argument("--ttl", type=int, default=int(os.environ.get("LAB3_TTL", saved.get("SESSION_TTL_SECONDS", "3600"))))
|
||||
parser.add_argument("--test-port", type=int, default=18113)
|
||||
args = parser.parse_args()
|
||||
if args.project != "kuznetsov-lab3" and not args.project.startswith("kuznetsov-lab3-"):
|
||||
parser.error("Use a dedicated Kuznetsov lab3 project")
|
||||
if not 1 <= args.port <= 65535 or not 1 <= args.test_port <= 65535 or not 1 <= args.ttl <= 86400:
|
||||
parser.error("Invalid port or session TTL")
|
||||
if args.action in ("setup", "build", "start", "test", "restart-check", "failover-check"):
|
||||
setup(args)
|
||||
actions = {"setup": lambda item: None, "build": build, "start": start,
|
||||
"test": lambda item: (contract_tests(item), api_tests(item)),
|
||||
"restart": restart, "restart-check": restart_check, "verify": verify,
|
||||
"stop-one": stop_one, "restore": restore, "failover-check": failover_check,
|
||||
"stop": lambda item: compose(item, "stop", "--timeout", "10", "lb", "api1", "api2", "db")}
|
||||
actions[args.action](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (RuntimeError, OSError, ValueError) as error:
|
||||
print("FAIL: " + str(error), file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
python "%~dp0lab.py" restart
|
||||
exit /b %errorlevel%
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
exec python3 "$TASK_DIR/lab.py" restart
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
python "%~dp0lab.py" stop-one
|
||||
exit /b %errorlevel%
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}"
|
||||
TASK_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
exec python3 "$TASK_DIR/lab.py" stop-one
|
||||
Reference in New Issue
Block a user