forked from mtusi2026/mtusi2026lab2
264 lines
9.2 KiB
Python
Executable File
264 lines
9.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
Client, Failure, check = client.Client, client.Failure, client.check
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def compose(*args, stdin=None):
|
|
cmd = ["docker", "compose", *args]
|
|
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
|
|
|
|
def containers(service):
|
|
ids = compose("ps", "-aq", service).stdout.split()
|
|
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
|
|
template = (
|
|
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
|
|
" {{.State.StartedAt}} {{.State.FinishedAt}}"
|
|
)
|
|
out = subprocess.run(
|
|
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
|
|
)
|
|
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
|
|
|
|
|
|
def upload(c, actor, chat, expected=202):
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
|
schema = "Attachment" if expected == 202 else None
|
|
return c.call(
|
|
"POST",
|
|
path,
|
|
expected,
|
|
actor=actor,
|
|
raw=FIXTURE,
|
|
headers={"Content-Type": "image/png"},
|
|
schema=schema,
|
|
)
|
|
|
|
|
|
def content(c, actor, attachment, expected=200):
|
|
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
|
|
|
|
|
|
def timed(fn):
|
|
started = time.monotonic()
|
|
try:
|
|
response = fn()
|
|
except Failure as exc:
|
|
return {
|
|
"status": "failure",
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"error": str(exc),
|
|
}
|
|
seconds = round(time.monotonic() - started, 3)
|
|
return {
|
|
"status": response["status"],
|
|
"seconds": seconds,
|
|
"instance": response["headers"].get("X-Instance-Id"),
|
|
}
|
|
|
|
|
|
def wait(fn, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
try:
|
|
return fn()
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(0.2)
|
|
|
|
|
|
def failover_upload(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
sent = upload(c, alice, chat)
|
|
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
|
c.wait_ready(bob, att)
|
|
started = time.monotonic()
|
|
hook = subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
|
|
hook_seconds = time.monotonic() - started
|
|
got = wait(lambda: content(c, bob, att), 15)
|
|
recovered = time.monotonic() - started
|
|
served = got["headers"].get("X-Instance-Id")
|
|
check(served != target, "download served by the stopped replica")
|
|
during = containers(args.service)
|
|
states = {item["hostname"]: item["status"] for item in during}
|
|
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
|
|
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
|
|
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
|
check(
|
|
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
|
"sha256 mismatch after failover",
|
|
)
|
|
if args.restore:
|
|
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
|
|
return {
|
|
"scenario": "failover-upload",
|
|
"upload_instance": target,
|
|
"download_instance": served,
|
|
"sha256_equal": True,
|
|
"hook_seconds": round(hook_seconds, 3),
|
|
"download_ok_after_seconds": round(recovered, 3),
|
|
"replicas_at_download": during,
|
|
"restored": args.restore,
|
|
"replicas_after_restore": containers(args.service) if args.restore else None,
|
|
}
|
|
|
|
|
|
def s3_outage(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
att = c.wait_ready(bob, upload(c, alice, chat)["json"])
|
|
check(
|
|
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
|
)
|
|
try:
|
|
during = {
|
|
"upload": timed(lambda: upload(c, alice, chat, [503])),
|
|
"metadata": timed(
|
|
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
|
|
),
|
|
"download": timed(lambda: content(c, bob, att, 503)),
|
|
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
|
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
|
}
|
|
stopped = containers(args.s3_service)
|
|
finally:
|
|
started = time.monotonic()
|
|
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
|
recovered = round(time.monotonic() - started, 3)
|
|
got = wait(lambda: content(c, bob, att), 30)
|
|
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
|
expected = {"upload": 503, "download": 503, "text_message": 201}
|
|
check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}")
|
|
check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded")
|
|
return {
|
|
"scenario": "s3-outage",
|
|
"s3_container_during_outage": stopped,
|
|
"during_outage": during,
|
|
"expected": expected,
|
|
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
|
"s3_back_healthy_seconds": recovered,
|
|
"download_after_recovery": "sha256 equal",
|
|
}
|
|
|
|
|
|
def abort_upload(c, args):
|
|
alice, _, _, chat = c.bootstrap()
|
|
name = f"abort-{uuid.uuid4().hex}.png"
|
|
parts = urlsplit(c.base)
|
|
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
conn = factory(parts.netloc, timeout=10)
|
|
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
|
|
headers = {
|
|
"Authorization": f"Bearer {alice['token']}",
|
|
"Content-Type": "image/png",
|
|
"Content-Length": str(args.declared),
|
|
"Accept": "application/json",
|
|
}
|
|
for key, value in headers.items():
|
|
conn.putheader(key, value)
|
|
conn.endheaders()
|
|
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
|
|
conn.close()
|
|
time.sleep(args.settle)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
|
|
out = compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
"-U",
|
|
user,
|
|
"-d",
|
|
db,
|
|
"-v",
|
|
f"name={name}",
|
|
stdin=sql,
|
|
)
|
|
rows = out.stdout.split()
|
|
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
|
check(out.returncode == 0, "cannot inspect aborted upload")
|
|
check(counts.get("ready", 0) == 0, "aborted upload became ready")
|
|
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
|
return {
|
|
"scenario": "abort-upload",
|
|
"declared_bytes": args.declared,
|
|
"sent_bytes": args.sent,
|
|
"settle_seconds": args.settle,
|
|
"rows_by_status": counts,
|
|
"ready_rows": counts.get("ready", 0),
|
|
"api_still_serves": alive == 200,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
|
|
)
|
|
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--ca-file", default=None)
|
|
p.add_argument("--timeout", type=float, default=15)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--s3-service", default="s3")
|
|
p.add_argument(
|
|
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
|
|
)
|
|
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
|
|
p.add_argument("--sent", type=int, default=256 * 1024)
|
|
p.add_argument("--settle", type=float, default=3)
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
|
|
run = {
|
|
"failover-upload": failover_upload,
|
|
"s3-outage": s3_outage,
|
|
"abort-upload": abort_upload,
|
|
}[args.scenario]
|
|
commit = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False
|
|
)
|
|
stamp = {
|
|
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"commit": commit.stdout.decode().strip(),
|
|
}
|
|
try:
|
|
report = {"ok": True, **stamp, **run(c, args)}
|
|
except Failure as exc:
|
|
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|