forked from mtusi2026/mtusi2026lab2
reconcile-inject.sh wrote an orphan under a key whose uuid differed from the id it printed, so the demo could only compare counts; it now uses one uuid for both, and the dry-run check compares the found and the injected sets per class. The heal step returns the original bytes under one quarantined key and random bytes of the same length under another, and checks that only the first leaves the quarantine. The load phase sums rechecked over its passes, and a new step stops a pass hanging on a paused S3 with docker stop and checks exit code 143 within seconds and a free lock afterwards. reconcile_track.py cost clears the run files of the previous series and measures 1k, 10k, 30k and 100k with --deep everywhere. The scenarios record date, commit and docker inspect of the replicas or of S3 at the moment they are down. make load-evidence and make clean-clone-evidence rerun the message load and the clean clone block with raw output, and make upload-memory passes ARGS to reproduce the run directories. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
259 lines
8.8 KiB
Python
Executable File
259 lines
8.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
Client, Failure, check = client.Client, client.Failure, client.check
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def compose(*args, stdin=None):
|
|
cmd = ["docker", "compose", *args]
|
|
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
|
|
|
|
def containers(service):
|
|
ids = compose("ps", "-aq", service).stdout.split()
|
|
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
|
|
template = (
|
|
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
|
|
" {{.State.StartedAt}} {{.State.FinishedAt}}"
|
|
)
|
|
out = subprocess.run(
|
|
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
|
|
)
|
|
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
|
|
|
|
|
|
def upload(c, actor, chat, expected=201):
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
|
schema = "Attachment" if expected == 201 else None
|
|
return c.call(
|
|
"POST",
|
|
path,
|
|
expected,
|
|
actor=actor,
|
|
raw=FIXTURE,
|
|
headers={"Content-Type": "image/png"},
|
|
schema=schema,
|
|
)
|
|
|
|
|
|
def content(c, actor, attachment, expected=200):
|
|
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
|
|
|
|
|
|
def timed(fn):
|
|
started = time.monotonic()
|
|
try:
|
|
response = fn()
|
|
except Failure as exc:
|
|
return {
|
|
"status": "failure",
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"error": str(exc),
|
|
}
|
|
seconds = round(time.monotonic() - started, 3)
|
|
return {
|
|
"status": response["status"],
|
|
"seconds": seconds,
|
|
"instance": response["headers"].get("X-Instance-Id"),
|
|
}
|
|
|
|
|
|
def wait(fn, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
try:
|
|
return fn()
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(0.2)
|
|
|
|
|
|
def failover_upload(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
sent = upload(c, alice, chat)
|
|
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
|
started = time.monotonic()
|
|
hook = subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
|
|
hook_seconds = time.monotonic() - started
|
|
got = wait(lambda: content(c, bob, att), 15)
|
|
recovered = time.monotonic() - started
|
|
served = got["headers"].get("X-Instance-Id")
|
|
check(served != target, "download served by the stopped replica")
|
|
during = containers(args.service)
|
|
states = {item["hostname"]: item["status"] for item in during}
|
|
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
|
|
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
|
|
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
|
check(
|
|
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
|
"sha256 mismatch after failover",
|
|
)
|
|
if args.restore:
|
|
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
|
|
return {
|
|
"scenario": "failover-upload",
|
|
"upload_instance": target,
|
|
"download_instance": served,
|
|
"sha256_equal": True,
|
|
"hook_seconds": round(hook_seconds, 3),
|
|
"download_ok_after_seconds": round(recovered, 3),
|
|
"replicas_at_download": during,
|
|
"restored": args.restore,
|
|
"replicas_after_restore": containers(args.service) if args.restore else None,
|
|
}
|
|
|
|
|
|
def s3_outage(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
att = upload(c, alice, chat)["json"]
|
|
check(
|
|
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
|
)
|
|
try:
|
|
during = {
|
|
"upload": timed(lambda: upload(c, alice, chat, [503])),
|
|
"metadata": timed(
|
|
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
|
|
),
|
|
"download": timed(lambda: content(c, bob, att, 503)),
|
|
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
|
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
|
}
|
|
stopped = containers(args.s3_service)
|
|
finally:
|
|
started = time.monotonic()
|
|
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
|
recovered = round(time.monotonic() - started, 3)
|
|
got = wait(lambda: content(c, bob, att), 30)
|
|
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
|
expected = {"upload": 503, "download": 503, "text_message": 201}
|
|
return {
|
|
"scenario": "s3-outage",
|
|
"s3_container_during_outage": stopped,
|
|
"during_outage": during,
|
|
"expected": expected,
|
|
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
|
"s3_back_healthy_seconds": recovered,
|
|
"download_after_recovery": "sha256 equal",
|
|
}
|
|
|
|
|
|
def abort_upload(c, args):
|
|
alice, _, _, chat = c.bootstrap()
|
|
name = f"abort-{uuid.uuid4().hex}.png"
|
|
parts = urlsplit(c.base)
|
|
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
conn = factory(parts.netloc, timeout=10)
|
|
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
|
|
headers = {
|
|
"Authorization": f"Bearer {alice['token']}",
|
|
"Content-Type": "image/png",
|
|
"Content-Length": str(args.declared),
|
|
"Accept": "application/json",
|
|
}
|
|
for key, value in headers.items():
|
|
conn.putheader(key, value)
|
|
conn.endheaders()
|
|
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
|
|
conn.close()
|
|
time.sleep(args.settle)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
|
|
out = compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
"-U",
|
|
user,
|
|
"-d",
|
|
db,
|
|
"-v",
|
|
f"name={name}",
|
|
stdin=sql,
|
|
)
|
|
rows = out.stdout.split()
|
|
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
|
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
|
return {
|
|
"scenario": "abort-upload",
|
|
"declared_bytes": args.declared,
|
|
"sent_bytes": args.sent,
|
|
"settle_seconds": args.settle,
|
|
"rows_by_status": counts,
|
|
"ready_rows": counts.get("ready", 0),
|
|
"api_still_serves": alive == 200,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
|
|
)
|
|
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--ca-file", default=None)
|
|
p.add_argument("--timeout", type=float, default=15)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--s3-service", default="s3")
|
|
p.add_argument(
|
|
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
|
|
)
|
|
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
|
|
p.add_argument("--sent", type=int, default=256 * 1024)
|
|
p.add_argument("--settle", type=float, default=3)
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
|
|
run = {
|
|
"failover-upload": failover_upload,
|
|
"s3-outage": s3_outage,
|
|
"abort-upload": abort_upload,
|
|
}[args.scenario]
|
|
commit = subprocess.run(
|
|
["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, check=False
|
|
)
|
|
stamp = {
|
|
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
"commit": commit.stdout.decode().strip(),
|
|
}
|
|
try:
|
|
report = {"ok": True, **stamp, **run(c, args)}
|
|
except Failure as exc:
|
|
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|