forked from mtusi2026/mtusi2026lab3
ЛР 3: балансировка, идемпотентность и проверки на 4
This commit is contained in:
@@ -0,0 +1,35 @@
|
||||
"""Run against a separate API configured with SESSION_TTL_SECONDS=2."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.environ["EXPIRY_BASE_URL"])
|
||||
deadline = time.monotonic() + 30
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except AssertionError:
|
||||
check(time.monotonic() < deadline, "TTL API did not become ready")
|
||||
time.sleep(0.25)
|
||||
actor = c.user("expiry")
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
time.sleep(2.3)
|
||||
response = c.call("GET", "/api/v1/users/me", 401, actor=actor)
|
||||
report = {"session_ttl_seconds": 2, "wait_seconds": 2.3,
|
||||
"before_expiry_status": 200, "after_expiry_status": response["status"],
|
||||
"result": "PASS"}
|
||||
(ROOT / "evidence" / "expiry.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
print("PASS: TTL=2s, initial 200, after 2.3s -> 401; normal API TTL unchanged")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Repeatable API failures and sequential replacement under HTTP load."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from ops import ROOT, command, compose, containers, summary
|
||||
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def wait_instances(client, actor, expected, excluded=(), version=None):
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
seen = {}
|
||||
try:
|
||||
for _ in range(20):
|
||||
r = client.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
seen[r["headers"]["X-Instance-Id"]] = r["headers"].get("X-App-Version")
|
||||
if len(seen) >= expected and not any(any(s.startswith(x + "-") for x in excluded) for s in seen):
|
||||
if version is None or all(v == version for v in seen.values()):
|
||||
return seen
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(.2)
|
||||
raise AssertionError("Expected API replicas were not observed through the load balancer")
|
||||
|
||||
|
||||
def history(client, actor, chat):
|
||||
from urllib.parse import quote
|
||||
result, cursor = [], None
|
||||
while True:
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
if cursor:
|
||||
path += "&cursor=" + quote(cursor, safe="")
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
result.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return result
|
||||
|
||||
|
||||
def resource_sample():
|
||||
ids = [r["id"] for r in summary() if r["status"] == "running"]
|
||||
return [json.loads(line) for line in command(["docker", "stats", "--no-stream", "--format", "{{json .}}", *ids]).splitlines()]
|
||||
|
||||
|
||||
def workload(base_url, label, action, duration=24, concurrency=4):
|
||||
client = Client(base_url, timeout=3)
|
||||
alice, bob, _, chat = client.bootstrap()
|
||||
before_message = client.message(alice, chat, "confirmed before " + label)["json"]
|
||||
observed_before = wait_instances(client, alice, 2)
|
||||
started = time.monotonic()
|
||||
deadline = started + duration
|
||||
events, confirmations = [], {}
|
||||
action_result = {}
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker(index):
|
||||
local = Client(base_url, timeout=3)
|
||||
key = uuid4().hex
|
||||
attempt = 0
|
||||
while time.monotonic() < deadline:
|
||||
begin = time.monotonic()
|
||||
try:
|
||||
if index == 0 and attempt % 4 == 0:
|
||||
response = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
||||
kind = "read"
|
||||
else:
|
||||
response = local.message(alice, chat, "load-" + key, key=key)
|
||||
kind = "write"
|
||||
with lock:
|
||||
confirmations[response["json"]["id"]] = response["json"]
|
||||
key = uuid4().hex
|
||||
event = {"at": time.monotonic() - started, "ok": True, "kind": kind,
|
||||
"ms": (time.monotonic() - begin) * 1000,
|
||||
"instance": response["headers"]["X-Instance-Id"]}
|
||||
except Failure:
|
||||
event = {"at": time.monotonic() - started, "ok": False,
|
||||
"ms": (time.monotonic() - begin) * 1000}
|
||||
# Preserve the key after uncertain POST responses.
|
||||
with lock:
|
||||
events.append(event)
|
||||
attempt += 1
|
||||
|
||||
def disrupt():
|
||||
time.sleep(4)
|
||||
action_result["started_seconds"] = time.monotonic() - started
|
||||
action_result["detail"] = action(client, alice)
|
||||
action_result["finished_seconds"] = time.monotonic() - started
|
||||
action_result["resources_during_load"] = resource_sample()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=concurrency + 1) as pool:
|
||||
workers = [pool.submit(worker, i) for i in range(concurrency)]
|
||||
change = pool.submit(disrupt)
|
||||
for future in workers:
|
||||
future.result()
|
||||
change.result()
|
||||
elapsed = time.monotonic() - started
|
||||
rows = history(client, bob, chat)
|
||||
by_id = {r["id"]: r for r in rows}
|
||||
assert by_id[before_message["id"]] == before_message
|
||||
assert all(by_id.get(mid) == msg for mid, msg in confirmations.items()), "Confirmed write lost"
|
||||
assert len(rows) == len(by_id) == len({r["text"] for r in rows}), "Retry created duplicate messages"
|
||||
success = [e for e in events if e["ok"]]
|
||||
failure = [e for e in events if not e["ok"]]
|
||||
finish = action_result["finished_seconds"]
|
||||
writes_after = [e for e in success if e["at"] >= finish and e["kind"] == "write"]
|
||||
reads_after = [e for e in success if e["at"] >= finish and e["kind"] == "read"]
|
||||
assert writes_after and reads_after, "Read/write did not resume"
|
||||
recovery = min(e["at"] for e in writes_after) - action_result["started_seconds"]
|
||||
if label.startswith("failure"):
|
||||
assert recovery <= 10, "Failover exceeded 10 seconds"
|
||||
hostname = action_result["detail"]["hostname"]
|
||||
assert all(not e["instance"].startswith(hostname + "-") for e in writes_after[-20:])
|
||||
latencies = sorted(e["ms"] for e in success)
|
||||
def percentile(q):
|
||||
return round(latencies[max(0, math.ceil(len(latencies) * q) - 1)], 3)
|
||||
buckets = []
|
||||
for start in range(0, math.ceil(elapsed), 2):
|
||||
group = [e for e in events if start <= e["at"] < start + 2]
|
||||
buckets.append({"start_seconds": start, "rps": sum(e["ok"] for e in group) / 2,
|
||||
"errors": sum(not e["ok"] for e in group)})
|
||||
report = {"scenario": label, "utc": datetime.now(timezone.utc).isoformat(),
|
||||
"duration_seconds": round(elapsed, 3), "concurrency": concurrency,
|
||||
"successful_requests": len(success), "failed_attempts": len(failure),
|
||||
"error_ratio": round(len(failure) / len(events), 6),
|
||||
"successful_rps": round(len(success) / elapsed, 3),
|
||||
"successful_latency_ms": {"p50": percentile(.5), "p95": percentile(.95), "p99": percentile(.99)},
|
||||
"first_write_after_action_seconds": round(recovery, 3), "action": action_result,
|
||||
"confirmed_writes_checked": len(confirmations) + 1, "history_messages": len(rows),
|
||||
"unique_history_messages": len(by_id), "instances_before": list(observed_before),
|
||||
"series_2_seconds": buckets, "deployment_after": summary(),
|
||||
"note": "Closed-loop mixed HTTP load, 4 clients; failures are attempts, retries retain keys; latency covers successes only."}
|
||||
(ROOT / "evidence" / (label + ".json")).write_text(json.dumps(report, indent=2) + "\n")
|
||||
print(label + ": " + str(report["successful_rps"]) + " RPS; " + str(len(failure)) + " errors", flush=True)
|
||||
return report
|
||||
|
||||
|
||||
def failure_action(target):
|
||||
def stop(client, actor):
|
||||
begin = time.monotonic()
|
||||
command(["docker", "stop", "--time", "6", target["Id"]], timeout=10)
|
||||
row = json.loads(command(["docker", "inspect", target["Id"]]))[0]
|
||||
assert not row["State"]["Running"] and row["State"]["ExitCode"] == 0
|
||||
elapsed = time.monotonic() - begin
|
||||
assert elapsed <= 10
|
||||
return {"container": row["Id"], "hostname": row["Config"]["Hostname"],
|
||||
"stop_seconds": round(elapsed, 3), "exit_code": row["State"]["ExitCode"]}
|
||||
return stop
|
||||
|
||||
|
||||
def rolling_action(client, actor):
|
||||
original = containers()
|
||||
assert len(original) == 2
|
||||
updates = []
|
||||
env = dict(os.environ, APP_VERSION="lab3-v2")
|
||||
# Both migrations are additive. Check/replay once before replacing processes.
|
||||
migration = compose("run", "--rm", "--no-deps", "migrate")
|
||||
for old in original:
|
||||
address = next(n["IPAddress"] for name, n in old["NetworkSettings"]["Networks"].items() if name.endswith("_edge"))
|
||||
slot = compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "drain", address)
|
||||
command(["docker", "stop", "--time", "6", old["Id"]])
|
||||
command(["docker", "rm", old["Id"]])
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=2", "api", env=env)
|
||||
compose("run", "--rm", "--no-deps", "checks", "python", "scripts/ops.py", "ready", slot)
|
||||
current = containers()
|
||||
fresh = [r for r in current if r["Id"] not in {x["Id"] for x in original} and r["Id"] not in {x["new_container"] for x in updates}]
|
||||
assert len(fresh) == 1
|
||||
wait_instances(client, actor, 2, excluded=[old["Config"]["Hostname"]])
|
||||
updates.append({"old_container": old["Id"], "new_container": fresh[0]["Id"], "backend_slot": slot,
|
||||
"new_image": fresh[0]["Image"]})
|
||||
wait_instances(client, actor, 2, version="lab3-v2")
|
||||
return {"updates": updates, "migration": migration,
|
||||
"method": "drain, stop, remove and recreate each container; same application image, new APP_VERSION configuration"}
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
originals = containers()
|
||||
assert len(originals) == 2
|
||||
reports = []
|
||||
for index, target in enumerate(originals, 1):
|
||||
try:
|
||||
reports.append(workload(args.base_url, "failure-api-" + str(index), failure_action(target)))
|
||||
finally:
|
||||
command(["docker", "start", target["Id"]])
|
||||
c = Client(args.base_url)
|
||||
alice = c.user("restore")
|
||||
wait_instances(c, alice, 2)
|
||||
reports.append(workload(args.base_url, "rolling-update", rolling_action, duration=45))
|
||||
(ROOT / "evidence" / "experiments.json").write_text(json.dumps([
|
||||
{k: r[k] for k in ("scenario", "successful_rps", "failed_attempts", "first_write_after_action_seconds",
|
||||
"confirmed_writes_checked", "successful_latency_ms")} for r in reports], indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Seed an isolated synthetic chat, measure the actual keyset SQL, keep raw plans."""
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from app.config import secret
|
||||
from app.security import Cursors, stamp
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def walk(node):
|
||||
yield node
|
||||
for child in node.get("Plans", []):
|
||||
yield from walk(child)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
actor = c.user("explain")
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=actor,
|
||||
body={"title": "EXPLAIN 10000", "member_ids": [actor["id"]]}, schema="Chat")["json"]
|
||||
base = datetime.now(timezone.utc)
|
||||
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
|
||||
conn.execute("""INSERT INTO messages(id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), %s, %s, 'synthetic-' || i,
|
||||
%s + i * interval '1 microsecond'
|
||||
FROM generate_series(1, 10000) AS i""", (UUID(chat["id"]), UUID(actor["id"]), base))
|
||||
conn.execute("UPDATE chats SET last_message_at=%s WHERE id=%s",
|
||||
(base + timedelta(microseconds=10000), UUID(chat["id"])))
|
||||
conn.execute("ANALYZE messages")
|
||||
report = {"messages_in_chat": 10000, "page_limit": 50,
|
||||
"index": "messages_chat_created_id_idx", "cases": {}}
|
||||
text_plans = []
|
||||
cursors = Cursors(secret("CURSOR_KEY").encode())
|
||||
with psycopg.connect(secret("DATABASE_URL"), connect_timeout=2) as conn:
|
||||
for label, offset in (("first", 0), ("middle", 5000), ("late", 9900)):
|
||||
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id=%s"
|
||||
params = [UUID(chat["id"])]
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=50"
|
||||
if offset:
|
||||
boundary = conn.execute("SELECT created_at, id FROM messages WHERE chat_id=%s AND text=%s",
|
||||
(UUID(chat["id"]), "synthetic-" + str(offset))).fetchone()
|
||||
sql += " AND (created_at, id) > (%s, %s)"
|
||||
params.extend(boundary)
|
||||
token = cursors.encode("messages:" + chat["id"],
|
||||
{"created_at": stamp(boundary[0]), "id": str(boundary[1])})
|
||||
path += "&cursor=" + quote(token, safe="")
|
||||
sql += " ORDER BY created_at, id LIMIT %s"
|
||||
params.append(51)
|
||||
data = conn.execute("EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + sql, params).fetchone()[0][0]
|
||||
nodes = list(walk(data["Plan"]))
|
||||
check(not any(x["Node Type"] == "Seq Scan" for x in nodes), "Unexpected full scan in keyset query")
|
||||
check(any(x.get("Index Name") == report["index"] for x in nodes), "History index not used")
|
||||
check(data["Plan"]["Actual Rows"] == 51, "LIMIT was not respected")
|
||||
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
check(len(page["items"]) == 50 and page["next_cursor"], "HTTP page differs from measured SQL")
|
||||
expected = ["synthetic-" + str(i) for i in range(offset + 1, offset + 51)]
|
||||
check([x["text"] for x in page["items"]] == expected, "HTTP keyset boundary differs")
|
||||
report["cases"][label] = {"after_message": offset, "sql": sql, "plan": data}
|
||||
text_plans.append(label.upper() + "\n" + sql + "\n" + "\n".join(
|
||||
row[0] for row in conn.execute("EXPLAIN (ANALYZE, BUFFERS) " + sql, params).fetchall()))
|
||||
(ROOT / "evidence" / "explain.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
(ROOT / "evidence" / "explain.txt").write_text("\n\n".join(text_plans) + "\n")
|
||||
print("PASS: 10000 messages, first/middle/late pages use history index; 51 SQL rows for 50 HTTP items")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Create local synthetic credentials once without printing secret values."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
directory = ROOT / "secrets"
|
||||
directory.mkdir(exist_ok=True)
|
||||
for name in ("postgres_password", "redis_password", "cursor_key"):
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(secrets.token_hex(32) + "\n", encoding="utf-8", newline="\n")
|
||||
else:
|
||||
# Shell command substitution on Linux must not retain a Windows CR.
|
||||
value = path.read_text(encoding="utf-8").strip()
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
pg_password = (directory / "postgres_password").read_text().strip()
|
||||
redis_password = (directory / "redis_password").read_text().strip()
|
||||
for name, value in {
|
||||
"database_url": f"postgresql://messenger:{pg_password}@postgres:5432/messenger",
|
||||
"cache_url": f"redis://:{redis_password}@redis:6379/0",
|
||||
}.items():
|
||||
path = directory / name
|
||||
if not path.exists():
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
env = ROOT / ".env"
|
||||
if not env.exists():
|
||||
env.write_bytes((ROOT / ".env.example").read_bytes())
|
||||
print("PASS: local .env and secret files exist; existing values preserved")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""Docker operations scoped to the configured local Compose project."""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, timeout=120, env=None):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout, env=env)
|
||||
if result.returncode:
|
||||
# Commands and stderr can contain secrets. Keep detailed errors local.
|
||||
raise RuntimeError("Local command failed: " + " ".join(args[:3]) + "\n" + result.stderr[-1500:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120, env=None):
|
||||
return command(["docker", "compose", "--env-file", ".env", *args], timeout, env)
|
||||
|
||||
|
||||
def project():
|
||||
return os.getenv("COMPOSE_PROJECT_NAME") or json.loads(compose("config", "--format", "json"))["name"]
|
||||
|
||||
|
||||
def containers(service="api", running=True):
|
||||
args = ["docker", "ps"] + ([] if running else ["-a"])
|
||||
ids = command(args + ["--filter", "label=com.docker.compose.project=" + project(),
|
||||
"--filter", "label=com.docker.compose.service=" + service,
|
||||
"--filter", "label=com.docker.compose.oneoff=False", "--format", "{{.ID}}"])
|
||||
return json.loads(command(["docker", "inspect", *ids.split()])) if ids else []
|
||||
|
||||
|
||||
def summary():
|
||||
result = []
|
||||
for service in ("api", "postgres", "redis", "lb"):
|
||||
for row in containers(service, running=False):
|
||||
result.append({"service": service, "id": row["Id"], "name": row["Name"].lstrip("/"),
|
||||
"started_at": row["State"]["StartedAt"], "status": row["State"]["Status"],
|
||||
"image": row["Image"], "user": row["Config"]["User"],
|
||||
"memory_bytes": row["HostConfig"]["Memory"], "nano_cpus": row["HostConfig"]["NanoCpus"],
|
||||
"published_ports": row["HostConfig"]["PortBindings"],
|
||||
"volumes": [m["Name"] for m in row["Mounts"] if m["Type"] == "volume"]})
|
||||
return result
|
||||
|
||||
|
||||
def runtime(line):
|
||||
with socket.create_connection(("lb", 9999), timeout=3) as sock:
|
||||
sock.sendall((line + "\n").encode())
|
||||
sock.shutdown(socket.SHUT_WR)
|
||||
chunks = []
|
||||
while True:
|
||||
data = sock.recv(65536)
|
||||
if not data:
|
||||
return b"".join(chunks).decode()
|
||||
chunks.append(data)
|
||||
|
||||
|
||||
def drain(address):
|
||||
lines = runtime("show servers state messenger").splitlines()
|
||||
fields = next(line[2:].split() for line in lines if line.startswith("# "))
|
||||
for line in lines:
|
||||
values = line.split()
|
||||
if not values or line.startswith("#") or len(values) != len(fields):
|
||||
continue
|
||||
row = dict(zip(fields, values))
|
||||
if row["srv_addr"] == address:
|
||||
slot = "messenger/" + row["srv_name"]
|
||||
runtime("set server " + slot + " state drain")
|
||||
# Let requests already assigned to this replica finish (HTTP budget 5 s).
|
||||
deadline = time.monotonic() + 7
|
||||
while time.monotonic() < deadline:
|
||||
stats = list(csv.DictReader(io.StringIO(runtime("show stat").lstrip("# "))))
|
||||
active = [s for s in stats if s["pxname"] == "messenger" and s["svname"] == row["srv_name"]]
|
||||
if active and int(active[0]["scur"]) == 0:
|
||||
return slot
|
||||
time.sleep(.1)
|
||||
raise RuntimeError("Replica did not drain within 7 seconds")
|
||||
raise RuntimeError("Backend address was not found in HAProxy")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
if sys.argv[1] == "drain":
|
||||
print(drain(sys.argv[2]))
|
||||
elif sys.argv[1] == "ready":
|
||||
runtime("set server " + sys.argv[2] + " state ready")
|
||||
elif sys.argv[1] == "summary":
|
||||
print(json.dumps(summary(), indent=2))
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose --env-file .env restart --timeout 6 api postgres redis lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python scripts/stop_one.py
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Stop exactly the replica selected by the public failover scenario."""
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from ops import ROOT, command, containers
|
||||
|
||||
|
||||
def main():
|
||||
target = os.environ["TARGET_INSTANCE_ID"]
|
||||
matches = [row for row in containers() if target.startswith(row["Config"]["Hostname"] + "-")]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Target does not identify exactly one running API replica")
|
||||
row = matches[0]
|
||||
started = time.monotonic()
|
||||
command(["docker", "stop", "--time", "6", row["Id"]], timeout=9)
|
||||
state = json.loads(command(["docker", "inspect", row["Id"]]))[0]["State"]
|
||||
assert not state["Running"] and state["ExitCode"] == 0, "Graceful shutdown failed"
|
||||
report = {"target_instance": target, "container": row["Id"], "exit_code": state["ExitCode"],
|
||||
"stop_seconds": round(time.monotonic() - started, 3), "remains_stopped": True}
|
||||
(ROOT / "evidence" / "last-stop.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Build and verify the complete local lab3 stack without deleting volumes."""
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from ops import ROOT, command, compose, containers, summary
|
||||
|
||||
|
||||
def saved(name, args, timeout=300):
|
||||
result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", errors="replace", timeout=timeout)
|
||||
(ROOT / "evidence" / name).write_text(result.stdout + result.stderr, encoding="utf-8")
|
||||
if result.returncode:
|
||||
raise RuntimeError("Check failed; see evidence/" + name)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def checks(name, *args):
|
||||
return saved(name, ["docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "checks", *args])
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--skip-build", action="store_true")
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--skip-experiments", action="store_true")
|
||||
args = p.parse_args()
|
||||
(ROOT / "evidence").mkdir(exist_ok=True)
|
||||
command([sys.executable, "-X", "utf8", "scripts/init_local.py"])
|
||||
compose("config", "--quiet")
|
||||
if not args.skip_build:
|
||||
print("Building API and check images", flush=True)
|
||||
saved("build.txt", ["docker", "compose", "--env-file", ".env", "build", "api", "checks"], 900)
|
||||
saved("startup.txt", ["docker", "compose", "--env-file", ".env", "up", "-d", "--wait", "--wait-timeout", "120"], 240)
|
||||
config = json.loads(compose("config", "--format", "json"))
|
||||
base_url = args.base_url or "http://127.0.0.1:" + str(config["services"]["lb"]["ports"][0]["published"])
|
||||
print("Checking contract, HTTP API and student tests", flush=True)
|
||||
checks("contract-check.txt", "make", "check")
|
||||
checks("smoke.txt", "make", "test")
|
||||
checks("student-tests.txt", "make", "test-student")
|
||||
before = summary()
|
||||
checks("persistence.txt", "sh", "-ec", "chmod +x scripts/restart.sh; make test-persistence ACTION_SCRIPT=scripts/restart.sh")
|
||||
after = summary()
|
||||
by_id = {r["id"]: r for r in after}
|
||||
for old in before:
|
||||
new = by_id[old["id"]]
|
||||
assert old["started_at"] != new["started_at"]
|
||||
assert old["volumes"] == new["volumes"]
|
||||
(ROOT / "evidence" / "restart.json").write_text(json.dumps({"before": before, "after": after}, indent=2) + "\n")
|
||||
print("Running public failover scenario", flush=True)
|
||||
try:
|
||||
checks("failover.txt", "sh", "-ec", "chmod +x scripts/stop-one.sh; make test-failover ACTION_SCRIPT=scripts/stop-one.sh")
|
||||
finally:
|
||||
compose("start", "api")
|
||||
# Wait for both replicas through LB before starting subsequent checks.
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
from experiments import wait_instances
|
||||
client = Client(base_url)
|
||||
actor = client.user("verify")
|
||||
wait_instances(client, actor, 2)
|
||||
print("Checking short session expiry and indexed history", flush=True)
|
||||
ttl_name = config["name"] + "-ttl-check"
|
||||
assert not command(["docker", "ps", "-a", "--filter", "name=^/" + ttl_name + "$", "--format", "{{.ID}}"])
|
||||
try:
|
||||
compose("run", "-d", "--no-deps", "--name", ttl_name, "-e", "SESSION_TTL_SECONDS=2", "api")
|
||||
checks("expiry.txt", "env", "EXPIRY_BASE_URL=http://" + ttl_name + ":8080", "python", "scripts/check_expiry.py")
|
||||
finally:
|
||||
command(["docker", "stop", "--time", "6", ttl_name])
|
||||
command(["docker", "rm", ttl_name])
|
||||
checks("explain-run.txt", "make", "explain")
|
||||
checks("smoke-final.txt", "make", "test")
|
||||
saved("published-smoke.txt", [sys.executable, "-X", "utf8", "tests/smoke.py", "--base-url", base_url])
|
||||
if not args.skip_experiments:
|
||||
print("Measuring failure of each replica and sequential replacement under load", flush=True)
|
||||
saved("experiments.txt", [sys.executable, "-X", "utf8", "scripts/experiments.py", "--base-url", base_url], 360)
|
||||
deployed = summary()
|
||||
assert len([r for r in deployed if r["service"] == "api" and r["status"] == "running"]) == 2
|
||||
assert all(not r["published_ports"] for r in deployed if r["service"] != "lb")
|
||||
assert all(r["memory_bytes"] and r["nano_cpus"] for r in deployed)
|
||||
versions = command(["docker", "info", "--format", "{{json .}}"])
|
||||
info = json.loads(versions)
|
||||
report = {"result": "PASS", "utc": datetime.now(timezone.utc).isoformat(), "base_url": base_url,
|
||||
"checks": ["public_smoke", "student_tests", "full_restart", "public_failover", "expiry", "index_10000", "published_smoke"]
|
||||
+ ([] if args.skip_experiments else ["each_replica_failure", "sequential_replacement_under_load"]),
|
||||
"docker": {k: info[k] for k in ("ServerVersion", "OperatingSystem", "Architecture", "NCPU", "MemTotal")},
|
||||
"deployment": deployed}
|
||||
(ROOT / "evidence" / "verification.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
print("PASS: lab3 checks completed; evidence/verification.json", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user