Files
mtusi2026lab2/scripts/lab4_scenarios.py
T
ArtyomandClaude Opus 5.5 fe2693870e Add the level 3 pipeline scenarios and the stand evidence
Bring the stdlib suite and image generator of the lab 5 preparation:
make suite-preflight checks services, readiness, schema columns, broker
topology and the faults override, and make level3 runs the grade 3
scenarios (smoke through the pipeline, worker down with API, relay and
broker restarts, duplicate delivery after ready, poison image, S3
outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the
invariants into evidence/level3.json. make pipeline-evidence shows the
queued job on the broker while the worker is stopped, its properties
and contract payload, the 409 before ready and SIGTERM of the relay and
the worker; make topology dumps queues, consumers with prefetch,
permissions and users.

The lab 4 failover and S3 outage scenarios and the upload fault suite
now expect 202 and wait for ready, the stack evidence lists the broker
and its client packages, and the CI acceptance job runs the pipeline
scenarios and the SIGTERM check of the new processes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 01:19:13 +03:00

232 lines
7.6 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import hashlib
import http.client
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
Client, Failure, check = client.Client, client.Failure, client.check
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def compose(*args, stdin=None):
cmd = ["docker", "compose", *args]
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
def upload(c, actor, chat, expected=202):
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
schema = "Attachment" if expected == 202 else None
return c.call(
"POST",
path,
expected,
actor=actor,
raw=FIXTURE,
headers={"Content-Type": "image/png"},
schema=schema,
)
def content(c, actor, attachment, expected=200):
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
def timed(fn):
started = time.monotonic()
try:
response = fn()
except Failure as exc:
return {
"status": "failure",
"seconds": round(time.monotonic() - started, 3),
"error": str(exc),
}
seconds = round(time.monotonic() - started, 3)
return {
"status": response["status"],
"seconds": seconds,
"instance": response["headers"].get("X-Instance-Id"),
}
def wait(fn, seconds):
deadline = time.monotonic() + seconds
while True:
try:
return fn()
except Failure:
if time.monotonic() > deadline:
raise
time.sleep(0.2)
def failover_upload(c, args):
alice, bob, _, chat = c.bootstrap()
sent = upload(c, alice, chat)
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
c.wait_ready(bob, att)
started = time.monotonic()
hook = subprocess.run(
[str(ROOT / "scripts/stop-one.sh")],
cwd=ROOT,
env=dict(os.environ, TARGET_INSTANCE_ID=target),
capture_output=True,
text=True,
check=False,
)
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
hook_seconds = time.monotonic() - started
got = wait(lambda: content(c, bob, att), 15)
recovered = time.monotonic() - started
served = got["headers"].get("X-Instance-Id")
check(served != target, "download served by the stopped replica")
digest = hashlib.sha256(got["bytes"]).hexdigest()
check(
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
"sha256 mismatch after failover",
)
if args.restore:
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
return {
"scenario": "failover-upload",
"upload_instance": target,
"download_instance": served,
"sha256_equal": True,
"hook_seconds": round(hook_seconds, 3),
"download_ok_after_seconds": round(recovered, 3),
"restored": args.restore,
}
def s3_outage(c, args):
alice, bob, _, chat = c.bootstrap()
att = c.wait_ready(bob, upload(c, alice, chat)["json"])
check(
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
)
try:
during = {
"upload": timed(lambda: upload(c, alice, chat, [503])),
"metadata": timed(
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
),
"download": timed(lambda: content(c, bob, att, 503)),
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
}
finally:
started = time.monotonic()
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
recovered = round(time.monotonic() - started, 3)
got = wait(lambda: content(c, bob, att), 30)
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
expected = {"upload": 503, "download": 503, "text_message": 201}
return {
"scenario": "s3-outage",
"during_outage": during,
"expected": expected,
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
"s3_back_healthy_seconds": recovered,
"download_after_recovery": "sha256 equal",
}
def abort_upload(c, args):
alice, _, _, chat = c.bootstrap()
name = f"abort-{uuid.uuid4().hex}.png"
parts = urlsplit(c.base)
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
conn = factory(parts.netloc, timeout=10)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
headers = {
"Authorization": f"Bearer {alice['token']}",
"Content-Type": "image/png",
"Content-Length": str(args.declared),
"Accept": "application/json",
}
for key, value in headers.items():
conn.putheader(key, value)
conn.endheaders()
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
conn.close()
time.sleep(args.settle)
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
out = compose(
"exec",
"-T",
"postgres",
"psql",
"-qAt",
"-F",
" ",
"-U",
user,
"-d",
db,
"-v",
f"name={name}",
stdin=sql,
)
rows = out.stdout.split()
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
return {
"scenario": "abort-upload",
"declared_bytes": args.declared,
"sent_bytes": args.sent,
"settle_seconds": args.settle,
"rows_by_status": counts,
"ready_rows": counts.get("ready", 0),
"api_still_serves": alive == 200,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
)
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
p.add_argument("--base-url", default=None)
p.add_argument("--ca-file", default=None)
p.add_argument("--timeout", type=float, default=15)
p.add_argument("--service", default="api")
p.add_argument("--s3-service", default="s3")
p.add_argument(
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
)
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
p.add_argument("--sent", type=int, default=256 * 1024)
p.add_argument("--settle", type=float, default=3)
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
run = {
"failover-upload": failover_upload,
"s3-outage": s3_outage,
"abort-upload": abort_upload,
}[args.scenario]
try:
report = {"ok": True, **run(c, args)}
except Failure as exc:
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
sys.exit(0 if report["ok"] else 1)
if __name__ == "__main__":
main()