119 lines
6.2 KiB
Python
119 lines
6.2 KiB
Python
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
from rolling_update import identity, stats
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, arguments, check, run
|
|
from resilience import history
|
|
|
|
COMPOSE = ["docker", "compose", "--project-name", "highload-lab3", "--env-file", ".env", "-f", "compose.yml"]
|
|
|
|
|
|
def inspect(container):
|
|
# Deliberately inspect only public identity/state, not credentials or env.
|
|
raw = subprocess.check_output(["docker", "inspect", "--format",
|
|
'{{json .Config.Hostname}} {{json .State.Status}}', container], text=True)
|
|
hostname, state = raw.strip().split()
|
|
return json.loads(hostname), json.loads(state)
|
|
|
|
|
|
def restore():
|
|
subprocess.run(COMPOSE + ["up", "-d", "--wait", "--no-deps", "api"], cwd=ROOT, check=True)
|
|
|
|
|
|
def main():
|
|
args = arguments(__doc__)
|
|
c = Client(args.base_url, args.ca_file, timeout=2)
|
|
containers = subprocess.check_output(COMPOSE + ["ps", "-q", "api"], cwd=ROOT, text=True).split()
|
|
check(len(containers) >= 2, "need at least two running API containers")
|
|
targets = [(container, inspect(container)[0]) for container in containers]
|
|
alice, bob, _, chat = c.bootstrap()
|
|
records = []
|
|
for container, target in targets:
|
|
# Read through the LB using one existing session until all replicas rejoin.
|
|
seen = set()
|
|
deadline = time.monotonic() + 10
|
|
while len(seen) < len(targets) and time.monotonic() < deadline:
|
|
seen.add(c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"])
|
|
time.sleep(0.05)
|
|
check(target in seen and len(seen) >= 2, "restored replica is not serving through HAProxy")
|
|
confirmed_key = uuid.uuid4().hex
|
|
confirmed_text = "Before failure " + confirmed_key
|
|
original = c.message(alice, chat, confirmed_text, key=confirmed_key)["json"]
|
|
env = dict(os.environ, TARGET_INSTANCE_ID=target)
|
|
addresses = {network["IPAddress"] for network in identity(container)["networks"].values()}
|
|
slot = next(row["svname"] for row in stats() if row["addr"].split(":")[0] in addresses)
|
|
started = time.monotonic()
|
|
proc = subprocess.Popen([str(ROOT / "scripts" / "stop-one.sh")], cwd=ROOT, env=env,
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
successes, last_instances = [], []
|
|
write_errors = read_errors = reads = 0
|
|
first_write = first_read = None
|
|
excluded = None
|
|
key = uuid.uuid4().hex
|
|
try:
|
|
while time.monotonic() - started < 15:
|
|
check(proc.poll() in (None, 0), "stop-one hook failed")
|
|
if excluded is None and any(row["svname"] == slot and not row["status"].startswith("UP") for row in stats()):
|
|
excluded = time.monotonic() - started
|
|
try:
|
|
result = c.message(alice, chat, "During failure " + key, key=key)
|
|
successes.append(result["json"])
|
|
if proc.poll() == 0:
|
|
last_instances.append(result["headers"]["X-Instance-Id"])
|
|
if first_write is None:
|
|
first_write = time.monotonic() - started
|
|
key = uuid.uuid4().hex
|
|
except Failure:
|
|
write_errors += 1 # Retry the SAME key and text on the next iteration.
|
|
try:
|
|
c.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")
|
|
reads += 1
|
|
if proc.poll() == 0 and first_read is None:
|
|
first_read = time.monotonic() - started
|
|
except Failure:
|
|
read_errors += 1
|
|
time.sleep(0.1)
|
|
elapsed = time.monotonic() - started
|
|
check(proc.poll() == 0, "stop-one hook did not finish")
|
|
check(inspect(container)[1] == "exited", "target container did not stay stopped")
|
|
check(first_write is not None and first_write <= 10, "writes did not recover within 10 seconds")
|
|
check(first_read is not None and first_read <= 10, "reads did not recover within 10 seconds")
|
|
check(excluded is not None and excluded <= 6, "HAProxy did not exclude stopped backend within the 6-second budget")
|
|
check(len(last_instances) >= 5 and target not in last_instances[-5:], "stopped replica still serves responses")
|
|
check(c.message(alice, chat, confirmed_text, key=confirmed_key)["json"] == original,
|
|
"replay after switching replicas changed a confirmed message")
|
|
messages = history(c, bob, chat)
|
|
check(all(message in messages for message in [original] + successes), "confirmed message lost")
|
|
texts = [message["text"] for message in messages]
|
|
check(len(texts) == len(set(texts)), "uncertain retry created a duplicate")
|
|
record = {"stopped_instance": target, "container_state": "exited", "successful_writes": len(successes),
|
|
"successful_reads": reads, "write_errors": write_errors, "read_errors": read_errors,
|
|
"first_write_seconds": round(first_write, 3), "first_read_seconds": round(first_read, 3)}
|
|
record.update(elapsed_seconds=round(elapsed, 3),
|
|
lb_exclusion_seconds=round(excluded, 3),
|
|
successful_rps=round((len(successes) + reads) / elapsed, 3),
|
|
attempted_rps=round((len(successes) + reads + write_errors + read_errors) / elapsed, 3),
|
|
workload="one sequential writer and reader, 100ms pause; not a capacity benchmark")
|
|
records.append(record)
|
|
print(json.dumps(record), flush=True)
|
|
finally:
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
proc.wait(timeout=5)
|
|
restore()
|
|
ROOT.joinpath(".memory/grade4").mkdir(parents=True, exist_ok=True)
|
|
ROOT.joinpath(".memory/grade4/failover.json").write_text(json.dumps(records, indent=2) + "\n")
|
|
print("PASS: each API replica stopped in turn; reads, writes, sessions and confirmed messages preserved")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|