#!/usr/bin/env python3 import argparse import hashlib import http.client import json import os import subprocess import sys import time import uuid from pathlib import Path from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) client = __import__("client") Client, Failure, check = client.Client, client.Failure, client.check FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() def compose(*args, stdin=None): cmd = ["docker", "compose", *args] return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False) def containers(service): ids = compose("ps", "-aq", service).stdout.split() fields = ("hostname", "status", "exit_code", "started_at", "finished_at") template = ( "{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}" " {{.State.StartedAt}} {{.State.FinishedAt}}" ) out = subprocess.run( ["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False ) return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()] def upload(c, actor, chat, expected=202): path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png" schema = "Attachment" if expected == 202 else None return c.call( "POST", path, expected, actor=actor, raw=FIXTURE, headers={"Content-Type": "image/png"}, schema=schema, ) def content(c, actor, attachment, expected=200): return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor) def timed(fn): started = time.monotonic() try: response = fn() except Failure as exc: return { "status": "failure", "seconds": round(time.monotonic() - started, 3), "error": str(exc), } seconds = round(time.monotonic() - started, 3) return { "status": response["status"], "seconds": seconds, "instance": response["headers"].get("X-Instance-Id"), } def wait(fn, seconds): deadline = time.monotonic() + seconds while True: try: return fn() except Failure: if time.monotonic() > deadline: raise time.sleep(0.2) def failover_upload(c, args): alice, bob, _, chat = c.bootstrap() sent = upload(c, alice, chat) att, target = sent["json"], sent["headers"]["X-Instance-Id"] c.wait_ready(bob, att) started = time.monotonic() hook = subprocess.run( [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT, env=dict(os.environ, TARGET_INSTANCE_ID=target), capture_output=True, text=True, check=False, ) check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}") hook_seconds = time.monotonic() - started got = wait(lambda: content(c, bob, att), 15) recovered = time.monotonic() - started served = got["headers"].get("X-Instance-Id") check(served != target, "download served by the stopped replica") during = containers(args.service) states = {item["hostname"]: item["status"] for item in during} check(states.get(target) == "exited", f"target {target} is not exited: {states}") check(states.get(served) == "running", f"neighbour {served} is not running: {states}") digest = hashlib.sha256(got["bytes"]).hexdigest() check( digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(), "sha256 mismatch after failover", ) if args.restore: compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service) return { "scenario": "failover-upload", "upload_instance": target, "download_instance": served, "sha256_equal": True, "hook_seconds": round(hook_seconds, 3), "download_ok_after_seconds": round(recovered, 3), "replicas_at_download": during, "restored": args.restore, "replicas_after_restore": containers(args.service) if args.restore else None, } def s3_outage(c, args): alice, bob, _, chat = c.bootstrap() att = c.wait_ready(bob, upload(c, alice, chat)["json"]) check( compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed" ) try: during = { "upload": timed(lambda: upload(c, alice, chat, [503])), "metadata": timed( lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob) ), "download": timed(lambda: content(c, bob, att, 503)), "text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")), "health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])), } stopped = containers(args.s3_service) finally: started = time.monotonic() compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service) recovered = round(time.monotonic() - started, 3) got = wait(lambda: content(c, bob, att), 30) check(got["bytes"] == FIXTURE, "attachment changed across S3 outage") expected = {"upload": 503, "download": 503, "text_message": 201} check(all(during[k]["status"] == v for k, v in expected.items()), f"S3 outage: {during}") check(all(during[k]["seconds"] < 6 for k in ("upload", "download")), "S3 timeout exceeded") return { "scenario": "s3-outage", "s3_container_during_outage": stopped, "during_outage": during, "expected": expected, "as_expected": all(during[k]["status"] == v for k, v in expected.items()), "s3_back_healthy_seconds": recovered, "download_after_recovery": "sha256 equal", } def abort_upload(c, args): alice, _, _, chat = c.bootstrap() name = f"abort-{uuid.uuid4().hex}.png" parts = urlsplit(c.base) factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection conn = factory(parts.netloc, timeout=10) conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}") headers = { "Authorization": f"Bearer {alice['token']}", "Content-Type": "image/png", "Content-Length": str(args.declared), "Accept": "application/json", } for key, value in headers.items(): conn.putheader(key, value) conn.endheaders() conn.send(FIXTURE[:8] + os.urandom(args.sent - 8)) conn.close() time.sleep(args.settle) user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n" out = compose( "exec", "-T", "postgres", "psql", "-qAt", "-F", " ", "-U", user, "-d", db, "-v", f"name={name}", stdin=sql, ) rows = out.stdout.split() counts = dict(zip(rows[::2], (int(x) for x in rows[1::2]))) check(out.returncode == 0, "cannot inspect aborted upload") check(counts.get("ready", 0) == 0, "aborted upload became ready") alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"] return { "scenario": "abort-upload", "declared_bytes": args.declared, "sent_bytes": args.sent, "settle_seconds": args.settle, "rows_by_status": counts, "ready_rows": counts.get("ready", 0), "api_still_serves": alive == 200, } def main(): p = argparse.ArgumentParser( description="Lab4 failure scenarios over the public HTTP contract plus docker compose" ) p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"]) p.add_argument("--base-url", default=None) p.add_argument("--ca-file", default=None) p.add_argument("--timeout", type=float, default=15) p.add_argument("--service", default="api") p.add_argument("--s3-service", default="s3") p.add_argument( "--restore", action="store_true", help="failover-upload: start the stopped replica again" ) p.add_argument("--declared", type=int, default=5 * 1024 * 1024) p.add_argument("--sent", type=int, default=256 * 1024) p.add_argument("--settle", type=float, default=3) p.add_argument("--output", metavar="FILE") args = p.parse_args() c = Client(args.base_url, args.ca_file, timeout=args.timeout) run = { "failover-upload": failover_upload, "s3-outage": s3_outage, "abort-upload": abort_upload, }[args.scenario] commit = subprocess.run( ["git", "rev-parse", "HEAD"], cwd=ROOT, capture_output=True, check=False ) stamp = { "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": commit.stdout.decode().strip(), } try: report = {"ok": True, **stamp, **run(c, args)} except Failure as exc: report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)} text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + "\n") print(text) sys.exit(0 if report["ok"] else 1) if __name__ == "__main__": main()