Files
mtusi2026lab4/scripts/lab4_scenarios.py
T
ArtyomandClaude Opus 5.5 f1ee385a1b Tie the track tooling to keys, sets and a clean series
reconcile-inject.sh wrote an orphan under a key whose uuid differed
from the id it printed, so the demo could only compare counts; it now
uses one uuid for both, and the dry-run check compares the found and
the injected sets per class. The heal step returns the original bytes
under one quarantined key and random bytes of the same length under
another, and checks that only the first leaves the quarantine. The
load phase sums rechecked over its passes, and a new step stops a pass
hanging on a paused S3 with docker stop and checks exit code 143 within
seconds and a free lock afterwards.

reconcile_track.py cost clears the run files of the previous series and
measures 1k, 10k, 30k and 100k with --deep everywhere. The scenarios
record date, commit and docker inspect of the replicas or of S3 at the
moment they are down. make load-evidence and make clean-clone-evidence
rerun the message load and the clean clone block with raw output, and
make upload-memory passes ARGS to reproduce the run directories.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-30 09:01:39 +03:00

259 lines
8.8 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
Client, Failure, check = client.Client, client.Failure, client.check
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def compose(*args, stdin=None):
cmd = ["docker", "compose", *args]
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def containers(service):
ids = compose("ps", "-aq", service).stdout.split()
fields = ("hostname", "status", "exit_code", "started_at", "finished_at")
template = (
"{{.Config.Hostname}} {{.State.Status}} {{.State.ExitCode}}"
" {{.State.StartedAt}} {{.State.FinishedAt}}"
)
out = subprocess.run(
["docker", "inspect", "-f", template, *ids], capture_output=True, text=True, check=False
)
return [dict(zip(fields, line.split())) for line in out.stdout.splitlines()]
def upload(c, actor, chat, expected=201):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 201 else None
return c.call(
"POST",
path,
expected,
actor=actor,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema=schema,
)
def content(c, actor, attachment, expected=200):
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
def timed(fn):
started = time.monotonic()
try:
response = fn()
except Failure as exc:
return {
"status": "failure",
"seconds": round(time.monotonic() - started, 3),
"error": str(exc),
}
seconds = round(time.monotonic() - started, 3)
return {
"status": response["status"],
"seconds": seconds,
"instance": response["headers"].get("X-Instance-Id"),
}
def wait(fn, seconds):
deadline = time.monotonic() + seconds
while True:
try:
return fn()
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.2)
def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
capture_output=True,
text=True,
check=False,
)
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
hook_seconds = time.monotonic() - started
got = wait(lambda: content(c, bob, att), 15)
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
during = containers(args.service)
states = {item["hostname"]: item["status"] for item in during}
check(states.get(target) == "exited", f"target {target} is not exited: {states}")
check(states.get(served) == "running", f"neighbour {served} is not running: {states}")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
"sha256 mismatch after failover",
)
if args.restore:
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
return {
"scenario": "failover-upload",
"upload_instance": target,
"download_instance": served,
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"replicas_at_download": during,
"restored": args.restore,
"replicas_after_restore": containers(args.service) if args.restore else None,
}
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat)["json"]
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
try:
during = {
"upload": timed(lambda: upload(c, alice, chat, [503])),
"metadata": timed(
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
),
"download": timed(lambda: content(c, bob, att, 503)),
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
stopped = containers(args.s3_service)
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
recovered = round(time.monotonic() - started, 3)
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
return {
"scenario": "s3-outage",
"s3_container_during_outage": stopped,
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
"s3_back_healthy_seconds": recovered,
"download_after_recovery": "sha256 equal",
}
def abort_upload(c, args):
alice, _, _, chat = c.bootstrap()
name = f"abort-{uuid.uuid4().hex}.png"
parts = urlsplit(c.base)
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
conn = factory(parts.netloc, timeout=10)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
headers = {
"Authorization": f"Bearer {alice['token']}",
"Content-Type": "image/png",
"Content-Length": str(args.declared),
"Accept": "application/json",
}
for key, value in headers.items():
conn.putheader(key, value)
conn.endheaders()
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
conn.close()
time.sleep(args.settle)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
out = compose(
"exec",
"-T",
"postgres",
"psql",
"-qAt",
"-F",
" ",
"-U",
user,
"-d",
db,
"-v",
f"name={name}",
stdin=sql,
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
"declared_bytes": args.declared,
"sent_bytes": args.sent,
"settle_seconds": args.settle,
"rows_by_status": counts,
"ready_rows": counts.get("ready", 0),
"api_still_serves": alive == 200,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
)
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
p.add_argument("--timeout", type=float, default=15)
p.add_argument("--service", default="api")
p.add_argument("--s3-service", default="s3")
p.add_argument(
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
)
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
p.add_argument("--sent", type=int, default=256 * 1024)
p.add_argument("--settle", type=float, default=3)
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
run = {
"failover-upload": failover_upload,
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
commit = subprocess.run(
["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, check=False
)
stamp = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"commit": commit.stdout.decode().strip(),
}
try:
report = {"ok": True, **stamp, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, **stamp, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()