Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6 and RustFS configuration, migrations, hooks, evidence and
scenario scripts and the CI job from the lab 4 solution commit d2744bc,
merge the lab 4 Makefile targets into the lab 5 skeleton Makefile
without touching its targets, and append the lab 4 solution variables
to the skeleton .env.example. The package and the application follow
the contract version 1.5.0, and the image and the throwaway test
containers are tagged lab5 so they do not collide with a lab 4 stand on
the same host.
This is the starting point of the lab 5 cycle: lint, own tests and the
offline skeleton check pass before any lab 5 change.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
231 lines
7.5 KiB
Python
Executable File
231 lines
7.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
client = __import__("client")
|
|
Client, Failure, check = client.Client, client.Failure, client.check
|
|
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def compose(*args, stdin=None):
|
|
cmd = ["docker", "compose", *args]
|
|
return subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, text=True, check=False)
|
|
|
|
|
|
def upload(c, actor, chat, expected=201):
|
|
path = f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png"
|
|
schema = "Attachment" if expected == 201 else None
|
|
return c.call(
|
|
"POST",
|
|
path,
|
|
expected,
|
|
actor=actor,
|
|
raw=FIXTURE,
|
|
headers={"Content-Type": "image/png"},
|
|
schema=schema,
|
|
)
|
|
|
|
|
|
def content(c, actor, attachment, expected=200):
|
|
return c.call("GET", f"/api/v1/attachments/{attachment['id']}/content", expected, actor=actor)
|
|
|
|
|
|
def timed(fn):
|
|
started = time.monotonic()
|
|
try:
|
|
response = fn()
|
|
except Failure as exc:
|
|
return {
|
|
"status": "failure",
|
|
"seconds": round(time.monotonic() - started, 3),
|
|
"error": str(exc),
|
|
}
|
|
seconds = round(time.monotonic() - started, 3)
|
|
return {
|
|
"status": response["status"],
|
|
"seconds": seconds,
|
|
"instance": response["headers"].get("X-Instance-Id"),
|
|
}
|
|
|
|
|
|
def wait(fn, seconds):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
try:
|
|
return fn()
|
|
except Failure:
|
|
if time.monotonic() > deadline:
|
|
raise
|
|
time.sleep(0.2)
|
|
|
|
|
|
def failover_upload(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
sent = upload(c, alice, chat)
|
|
att, target = sent["json"], sent["headers"]["X-Instance-Id"]
|
|
started = time.monotonic()
|
|
hook = subprocess.run(
|
|
[str(ROOT / "scripts/stop-one.sh")],
|
|
cwd=ROOT,
|
|
env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
|
capture_output=True,
|
|
text=True,
|
|
check=False,
|
|
)
|
|
check(hook.returncode == 0, f"stop-one.sh failed: {hook.stderr.strip()}")
|
|
hook_seconds = time.monotonic() - started
|
|
got = wait(lambda: content(c, bob, att), 15)
|
|
recovered = time.monotonic() - started
|
|
served = got["headers"].get("X-Instance-Id")
|
|
check(served != target, "download served by the stopped replica")
|
|
digest = hashlib.sha256(got["bytes"]).hexdigest()
|
|
check(
|
|
digest == att["sha256"] == hashlib.sha256(FIXTURE).hexdigest(),
|
|
"sha256 mismatch after failover",
|
|
)
|
|
if args.restore:
|
|
compose("up", "-d", "--wait", "--no-recreate", "--no-deps", args.service)
|
|
return {
|
|
"scenario": "failover-upload",
|
|
"upload_instance": target,
|
|
"download_instance": served,
|
|
"sha256_equal": True,
|
|
"hook_seconds": round(hook_seconds, 3),
|
|
"download_ok_after_seconds": round(recovered, 3),
|
|
"restored": args.restore,
|
|
}
|
|
|
|
|
|
def s3_outage(c, args):
|
|
alice, bob, _, chat = c.bootstrap()
|
|
att = upload(c, alice, chat)["json"]
|
|
check(
|
|
compose("stop", "-t", "10", args.s3_service).returncode == 0, "docker compose stop failed"
|
|
)
|
|
try:
|
|
during = {
|
|
"upload": timed(lambda: upload(c, alice, chat, [503])),
|
|
"metadata": timed(
|
|
lambda: c.call("GET", f"/api/v1/attachments/{att['id']}", [200, 503], actor=bob)
|
|
),
|
|
"download": timed(lambda: content(c, bob, att, 503)),
|
|
"text_message": timed(lambda: c.message(alice, chat, "S3 недоступен, текст работает")),
|
|
"health_ready": timed(lambda: c.call("GET", "/health/ready", [200, 503])),
|
|
}
|
|
finally:
|
|
started = time.monotonic()
|
|
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", args.s3_service)
|
|
recovered = round(time.monotonic() - started, 3)
|
|
got = wait(lambda: content(c, bob, att), 30)
|
|
check(got["bytes"] == FIXTURE, "attachment changed across S3 outage")
|
|
expected = {"upload": 503, "download": 503, "text_message": 201}
|
|
return {
|
|
"scenario": "s3-outage",
|
|
"during_outage": during,
|
|
"expected": expected,
|
|
"as_expected": all(during[k]["status"] == v for k, v in expected.items()),
|
|
"s3_back_healthy_seconds": recovered,
|
|
"download_after_recovery": "sha256 equal",
|
|
}
|
|
|
|
|
|
def abort_upload(c, args):
|
|
alice, _, _, chat = c.bootstrap()
|
|
name = f"abort-{uuid.uuid4().hex}.png"
|
|
parts = urlsplit(c.base)
|
|
factory = http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
conn = factory(parts.netloc, timeout=10)
|
|
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={name}")
|
|
headers = {
|
|
"Authorization": f"Bearer {alice['token']}",
|
|
"Content-Type": "image/png",
|
|
"Content-Length": str(args.declared),
|
|
"Accept": "application/json",
|
|
}
|
|
for key, value in headers.items():
|
|
conn.putheader(key, value)
|
|
conn.endheaders()
|
|
conn.send(FIXTURE[:8] + os.urandom(args.sent - 8))
|
|
conn.close()
|
|
time.sleep(args.settle)
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
sql = "SELECT status, count(*) FROM attachments WHERE filename = :'name' GROUP BY status;\n"
|
|
out = compose(
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
"-U",
|
|
user,
|
|
"-d",
|
|
db,
|
|
"-v",
|
|
f"name={name}",
|
|
stdin=sql,
|
|
)
|
|
rows = out.stdout.split()
|
|
counts = dict(zip(rows[::2], (int(x) for x in rows[1::2])))
|
|
alive = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["status"]
|
|
return {
|
|
"scenario": "abort-upload",
|
|
"declared_bytes": args.declared,
|
|
"sent_bytes": args.sent,
|
|
"settle_seconds": args.settle,
|
|
"rows_by_status": counts,
|
|
"ready_rows": counts.get("ready", 0),
|
|
"api_still_serves": alive == 200,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 failure scenarios over the public HTTP contract plus docker compose"
|
|
)
|
|
p.add_argument("scenario", choices=["failover-upload", "s3-outage", "abort-upload"])
|
|
p.add_argument("--base-url", default=None)
|
|
p.add_argument("--ca-file", default=None)
|
|
p.add_argument("--timeout", type=float, default=15)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--s3-service", default="s3")
|
|
p.add_argument(
|
|
"--restore", action="store_true", help="failover-upload: start the stopped replica again"
|
|
)
|
|
p.add_argument("--declared", type=int, default=5 * 1024 * 1024)
|
|
p.add_argument("--sent", type=int, default=256 * 1024)
|
|
p.add_argument("--settle", type=float, default=3)
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
c = Client(args.base_url, args.ca_file, timeout=args.timeout)
|
|
run = {
|
|
"failover-upload": failover_upload,
|
|
"s3-outage": s3_outage,
|
|
"abort-upload": abort_upload,
|
|
}[args.scenario]
|
|
try:
|
|
report = {"ok": True, **run(c, args)}
|
|
except Failure as exc:
|
|
report = {"scenario": args.scenario, "ok": False, "error": str(exc)}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
sys.exit(0 if report["ok"] else 1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|