Bring the stdlib suite and image generator of the lab 5 preparation: make suite-preflight checks services, readiness, schema columns, broker topology and the faults override, and make level3 runs the grade 3 scenarios (smoke through the pipeline, worker down with API, relay and broker restarts, duplicate delivery after ready, poison image, S3 outage with recovery, retries up to JOB_MAX_ATTEMPTS) and checks the invariants into evidence/level3.json. make pipeline-evidence shows the queued job on the broker while the worker is stopped, its properties and contract payload, the 409 before ready and SIGTERM of the relay and the worker; make topology dumps queues, consumers with prefetch, permissions and users. The lab 4 failover and S3 outage scenarios and the upload fault suite now expect 202 and wait for ready, the stack evidence lists the broker and its client packages, and the CI acceptance job runs the pipeline scenarios and the SIGTERM check of the new processes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
232 lines
7.6 KiB
Python
Executable File
232 lines
7.6 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
Client, Failure, check = client.Client, client.Failure, client.check
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def compose(*args, stdin=None):
|
|
cmd = ["docker", "compose", *args]
|
|
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
|
|
|
|
def upload(c, actor, chat, expected=202):
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
|
schema = "Attachment" if expected == 202 else None
|
|
return c.call(
|
|
"POST",
|
|
path,
|
|
expected,
|
|
actor=actor,
|
|
raw=FIXTURE,
|
|
headers={"Content-Type": "image/png"},
|
|
schema=schema,
|
|
)
|
|
|
|
|
|
def content(c, actor, attachment, expected=200):
|
|
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
|
|
|
|
|
|
def timed(fn):
|
|
started = time.monotonic()
|
|
try:
|
|
response = fn()
|
|
except Failure as exc:
|
|
return {
|
|
"status": "failure",
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"error": str(exc),
|
|
}
|
|
seconds = round(time.monotonic() - started, 3)
|
|
return {
|
|
"status": response["status"],
|
|
"seconds": seconds,
|
|
"instance": response["headers"].get("X-Instance-Id"),
|
|
}
|
|
|
|
|
|
def wait(fn, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
try:
|
|
return fn()
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(0.2)
|
|
|
|
|
|
def failover_upload(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
sent = upload(c, alice, chat)
|
|
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
|
c.wait_ready(bob, att)
|
|
started = time.monotonic()
|
|
hook = subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
|
|
hook_seconds = time.monotonic() - started
|
|
got = wait(lambda: content(c, bob, att), 15)
|
|
recovered = time.monotonic() - started
|
|
served = got["headers"].get("X-Instance-Id")
|
|
check(served != target, "download served by the stopped replica")
|
|
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
|
check(
|
|
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
|
"sha256 mismatch after failover",
|
|
)
|
|
if args.restore:
|
|
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
|
|
return {
|
|
"scenario": "failover-upload",
|
|
"upload_instance": target,
|
|
"download_instance": served,
|
|
"sha256_equal": True,
|
|
"hook_seconds": round(hook_seconds, 3),
|
|
"download_ok_after_seconds": round(recovered, 3),
|
|
"restored": args.restore,
|
|
}
|
|
|
|
|
|
def s3_outage(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
att = c.wait_ready(bob, upload(c, alice, chat)["json"])
|
|
check(
|
|
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
|
)
|
|
try:
|
|
during = {
|
|
"upload": timed(lambda: upload(c, alice, chat, [503])),
|
|
"metadata": timed(
|
|
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
|
|
),
|
|
"download": timed(lambda: content(c, bob, att, 503)),
|
|
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
|
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
|
}
|
|
finally:
|
|
started = time.monotonic()
|
|
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
|
recovered = round(time.monotonic() - started, 3)
|
|
got = wait(lambda: content(c, bob, att), 30)
|
|
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
|
expected = {"upload": 503, "download": 503, "text_message": 201}
|
|
return {
|
|
"scenario": "s3-outage",
|
|
"during_outage": during,
|
|
"expected": expected,
|
|
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
|
"s3_back_healthy_seconds": recovered,
|
|
"download_after_recovery": "sha256 equal",
|
|
}
|
|
|
|
|
|
def abort_upload(c, args):
|
|
alice, _, _, chat = c.bootstrap()
|
|
name = f"abort-{uuid.uuid4().hex}.png"
|
|
parts = urlsplit(c.base)
|
|
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
conn = factory(parts.netloc, timeout=10)
|
|
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
|
|
headers = {
|
|
"Authorization": f"Bearer {alice['token']}",
|
|
"Content-Type": "image/png",
|
|
"Content-Length": str(args.declared),
|
|
"Accept": "application/json",
|
|
}
|
|
for key, value in headers.items():
|
|
conn.putheader(key, value)
|
|
conn.endheaders()
|
|
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
|
|
conn.close()
|
|
time.sleep(args.settle)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
|
|
out = compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
"-U",
|
|
user,
|
|
"-d",
|
|
db,
|
|
"-v",
|
|
f"name={name}",
|
|
stdin=sql,
|
|
)
|
|
rows = out.stdout.split()
|
|
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
|
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
|
return {
|
|
"scenario": "abort-upload",
|
|
"declared_bytes": args.declared,
|
|
"sent_bytes": args.sent,
|
|
"settle_seconds": args.settle,
|
|
"rows_by_status": counts,
|
|
"ready_rows": counts.get("ready", 0),
|
|
"api_still_serves": alive == 200,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
|
|
)
|
|
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--ca-file", default=None)
|
|
p.add_argument("--timeout", type=float, default=15)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--s3-service", default="s3")
|
|
p.add_argument(
|
|
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
|
|
)
|
|
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
|
|
p.add_argument("--sent", type=int, default=256 * 1024)
|
|
p.add_argument("--settle", type=float, default=3)
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
|
|
run = {
|
|
"failover-upload": failover_upload,
|
|
"s3-outage": s3_outage,
|
|
"abort-upload": abort_upload,
|
|
}[args.scenario]
|
|
try:
|
|
report = {"ok": True, **run(c, args)}
|
|
except Failure as exc:
|
|
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|