forked from mtusi2026/mtusi2026lab5
369 lines
18 KiB
Python
369 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
"""Automatic fault injection for the attachment pipeline (levels 3-4 and track 5.2 "proved idempotency").
|
|
|
|
Requires the stack started with FAULT_INJECTION=1 (see REPORT.md). Controls containers via `docker compose`
|
|
and arms faults via the operator CLI; never deletes volumes. Each scenario ends with the invariant check:
|
|
|
|
I1 accepted work is not lost: every 202 attachment reaches its expected final state
|
|
I2 one result set: exactly one variant row per (attachment, version, variant) for the published version,
|
|
API metadata == DB rows, and bytes served by the API match the published sha256/size
|
|
I3 no duplicate effects: final S3 prefix == original + keys referenced by committed rows (+ explicitly
|
|
expected unreferenced leftovers of a superseded version)
|
|
I4 monotonic versions: 'ready' events never go back to an older pipeline_version; published version is the max
|
|
I5 nothing left behind: no job in processing, outbox fully published, staging removed after ready
|
|
|
|
python3 student-tests/faults_test.py # all scenarios
|
|
python3 student-tests/faults_test.py --only duplicate,reorder
|
|
"""
|
|
import hashlib
|
|
import json
|
|
import time
|
|
import uuid
|
|
from urllib.parse import quote
|
|
|
|
from stack import SAMPLE_PNG, arm, corrupt_png, dc, inspect, ops, stats, wait_for
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
SCENARIOS = []
|
|
|
|
|
|
def scenario(fn):
|
|
SCENARIOS.append(fn)
|
|
return fn
|
|
|
|
|
|
class Ctx:
|
|
def __init__(self, c, alice, bob, chat):
|
|
self.c, self.alice, self.bob, self.chat = c, alice, bob, chat
|
|
self.notes = []
|
|
|
|
def upload(self, data=SAMPLE_PNG, name="f.png"):
|
|
path = "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=" + quote(name, safe="")
|
|
att = self.c.call("POST", path, 202, actor=self.alice, raw=data, headers={"Content-Type": "image/png"},
|
|
schema="Attachment")["json"]
|
|
check(att["status"] == "queued", "upload must answer 202 queued")
|
|
return att
|
|
|
|
def get(self, att_id):
|
|
return self.c.call("GET", "/api/v1/attachments/" + att_id, actor=self.bob, schema="Attachment")["json"]
|
|
|
|
def wait(self, att_id, status="ready", timeout=90):
|
|
def done():
|
|
a = self.get(att_id)
|
|
if a["status"] in ("ready", "failed"):
|
|
return a
|
|
return None
|
|
a = wait_for(done, timeout, "final state of " + att_id, 0.3)
|
|
check(a["status"] == status, "expected %s, got %s (%s)" % (status, a["status"], a["error_code"]))
|
|
return a
|
|
|
|
def message(self, text, ids=None):
|
|
return self.c.message(self.alice, self.chat, text, attachments=ids)["json"]
|
|
|
|
def note(self, text):
|
|
self.notes.append(text)
|
|
print(" ", text)
|
|
|
|
|
|
def events(info, name=None):
|
|
return [e for e in info["events"] if name is None or e["event"] == name]
|
|
|
|
|
|
def check_invariants(ctx, att_id, expect_version=1, leftovers=()):
|
|
info = inspect(att_id)
|
|
att = ctx.get(att_id)
|
|
a = info["attachment"]
|
|
check(att["status"] == a["status"] == "ready", "I1: not ready")
|
|
published = a["pipeline_version"]
|
|
check(published == expect_version, "I4: published v%s, expected v%s" % (published, expect_version))
|
|
rows = [v for v in info["variants"] if v["pipeline_version"] == published]
|
|
names = sorted(v["name"] for v in rows)
|
|
check(names == ["crop", "optimized", "thumbnail"], "I2: variant rows %s" % names)
|
|
api = {v["name"]: v for v in att["variants"]}
|
|
for v in rows:
|
|
m = api[v["name"]]
|
|
check((m["sha256"], m["size_bytes"], m["width"], m["height"]) == (v["sha256"], v["size_bytes"], v["width"], v["height"]),
|
|
"I2: API metadata differs from DB row")
|
|
d = ctx.c.call("GET", "/api/v1/attachments/%s/content?variant=%s" % (att_id, v["name"]), actor=ctx.bob)
|
|
check(hashlib.sha256(d["bytes"]).hexdigest() == v["sha256"], "I2: served bytes differ from published sha256")
|
|
original = ctx.c.call("GET", "/api/v1/attachments/%s/content" % att_id, actor=ctx.bob)["bytes"]
|
|
check(hashlib.sha256(original).hexdigest() == a["sha256"], "I2: original changed")
|
|
expected = {"attachments/%s/original" % uuid.UUID(att_id).hex} | {v["object_key"] for v in info["variants"]} | set(leftovers)
|
|
check(set(info["s3_final_keys"]) == expected, "I3: S3 keys %s != %s" % (info["s3_final_keys"], sorted(expected)))
|
|
readies = [int(e["detail"].split(":")[0][1:]) for e in events(info, "ready")]
|
|
check(readies == sorted(readies) and readies[-1] == published, "I4: ready history %s" % readies)
|
|
check(all(j["status"] != "processing" for j in info["jobs"]), "I5: job left in processing")
|
|
check(all(o["published"] for o in info["outbox"]), "I5: unpublished outbox rows")
|
|
check(info["s3_staging_keys"] == [], "I5: staging not removed")
|
|
return info
|
|
|
|
|
|
def compose_ps(service):
|
|
return dc("ps", service, "--all", "--format", "{{.Name}} {{.State}} {{.Status}}").stdout.strip()
|
|
|
|
|
|
# ============================================================ level 3
|
|
@scenario
|
|
def worker_down(ctx):
|
|
"""worker stopped -> upload 202 -> message -> API + broker restart -> worker start -> ready"""
|
|
dc("stop", "worker")
|
|
ctx.note("worker: " + compose_ps("worker"))
|
|
att = ctx.upload(name="while-worker-down.png")
|
|
msg = ctx.message("файл, пока worker остановлен", [att["id"]])
|
|
check(msg["attachment_ids"] == [att["id"]], "queued attachment must be linkable")
|
|
ctx.c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=ctx.bob)
|
|
other = ctx.message("другие сообщения работают")
|
|
check(other["text"], "messaging works while the worker is down")
|
|
time.sleep(2)
|
|
s = stats()
|
|
check(s["queue_ready_messages"] >= 1 and s["queue_consumers"] == 0, "message must wait in the queue: %s" % s)
|
|
ctx.note("queue before broker restart: %s" % json.dumps(s))
|
|
dc("restart", "rabbitmq")
|
|
dc("up", "-d", "--wait", "--no-build", "rabbitmq", "relay")
|
|
s = wait_for(lambda: (lambda x: x if x["queue_ready_messages"] >= 1 else None)(stats()), 60, "queue after restart")
|
|
ctx.note("queue after broker restart (volume kept): %s" % json.dumps(s))
|
|
dc("restart", "api")
|
|
dc("up", "-d", "--wait", "--no-build", "api", "lb")
|
|
check(ctx.get(att["id"])["status"] == "queued", "still queued after API restart")
|
|
dc("up", "-d", "--wait", "--no-build", "worker")
|
|
ready = ctx.wait(att["id"])
|
|
thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0]
|
|
check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail size")
|
|
check_invariants(ctx, att["id"])
|
|
ctx.note("attachment %s ready after worker start; thumbnail 256x160" % att["id"])
|
|
|
|
|
|
# ============================================================ level 4: outbox, crashes, retries, lease, replay
|
|
@scenario
|
|
def outbox_gap(ctx):
|
|
"""COMMIT done, relay down (= crash between COMMIT and publish) -> relay restart publishes -> ready"""
|
|
dc("stop", "relay")
|
|
att = ctx.upload(name="outbox-gap.png")
|
|
time.sleep(1.5)
|
|
info = inspect(att["id"])
|
|
check([o["published"] for o in info["outbox"]] == [False], "outbox row must be pending while relay is down")
|
|
check(ctx.get(att["id"])["status"] == "queued", "queued")
|
|
ctx.note("relay down: outbox %s, attachment queued" % info["outbox"])
|
|
dc("up", "-d", "--wait", "--no-build", "relay")
|
|
ctx.wait(att["id"])
|
|
info = check_invariants(ctx, att["id"])
|
|
ctx.note("after relay start: ready, events %s" % [e["event"] for e in info["events"]])
|
|
|
|
|
|
@scenario
|
|
def relay_crash_after_confirm(ctx):
|
|
"""relay dies after the publisher confirm but before marking the row -> duplicate publish -> worker no-op"""
|
|
arm("relay.after_confirm_before_mark", "crash")
|
|
att = ctx.upload(name="relay-crash.png")
|
|
ctx.wait(att["id"])
|
|
info = wait_for(lambda: (lambda i: i if events(i, "duplicate_noop") else None)(inspect(att["id"])), 60,
|
|
"duplicate delivery")
|
|
info = check_invariants(ctx, att["id"])
|
|
check(len(events(info, "claimed")) == 1, "the duplicate must not start a second attempt")
|
|
ctx.note("events: %s" % [e["event"] for e in info["events"]])
|
|
|
|
|
|
def crash_scenario(ctx, point, name):
|
|
arm(point, "crash")
|
|
att = ctx.upload(name=name)
|
|
started = time.monotonic()
|
|
ctx.wait(att["id"], timeout=120)
|
|
info = check_invariants(ctx, att["id"])
|
|
ctx.note("%s: ready in %.1fs; events %s; variant attempts %s" % (
|
|
point, time.monotonic() - started, [e["event"] for e in info["events"]],
|
|
sorted({v["attempt"] for v in info["variants"]})))
|
|
return info
|
|
|
|
|
|
@scenario
|
|
def crash_after_claim(ctx):
|
|
"""crash before any side effect: lease expires, reaper re-enqueues attempt 2"""
|
|
info = crash_scenario(ctx, "worker.after_claim", "crash-claim.png")
|
|
check(events(info, "lease_expired") and info["jobs"][0]["attempt"] == 2, "recovered via lease expiry")
|
|
|
|
|
|
@scenario
|
|
def crash_after_s3(ctx):
|
|
"""crash after writing S3 objects, before COMMIT: retry overwrites the same keys, one set of variants"""
|
|
info = crash_scenario(ctx, "worker.after_s3_before_commit", "crash-s3.png")
|
|
check(events(info, "lease_expired") and {v["attempt"] for v in info["variants"]} == {2}, "result of attempt 2")
|
|
|
|
|
|
@scenario
|
|
def crash_after_commit(ctx):
|
|
"""crash after COMMIT, before ack: redelivery sees done -> ack as no-op, no second attempt"""
|
|
info = crash_scenario(ctx, "worker.after_commit_before_ack", "crash-ack.png")
|
|
check(events(info, "duplicate_noop"), "redelivered message must be a no-op")
|
|
check(info["jobs"][0]["attempt"] == 1 and len(events(info, "claimed")) == 1, "single attempt")
|
|
|
|
|
|
@scenario
|
|
def transient_backoff(ctx):
|
|
"""two transient failures -> retries with exponential backoff + jitter -> ready on attempt 3"""
|
|
arm("worker.before_transform", "transient", count=2)
|
|
att = ctx.upload(name="transient.png")
|
|
ctx.wait(att["id"])
|
|
info = check_invariants(ctx, att["id"])
|
|
delays = [float(e["detail"].rsplit(" in ", 1)[1].rstrip("s")) for e in events(info, "retry_scheduled")]
|
|
check(len(delays) == 2 and 1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, "backoff windows %s" % delays)
|
|
check(info["jobs"][0]["attempt"] == 3, "ready on attempt 3")
|
|
ctx.note("retry delays %s (windows [1,2] and [2,4] s)" % delays)
|
|
|
|
|
|
@scenario
|
|
def exhausted_then_replay(ctx):
|
|
"""5 transient failures -> failed retry_exhausted (registry) -> explicit replay -> ready; replay of ready refused"""
|
|
arm("worker.before_transform", "transient", count=5)
|
|
att = ctx.upload(name="exhausted.png")
|
|
failed = ctx.wait(att["id"], "failed", timeout=120)
|
|
check(failed["error_code"] == "retry_exhausted", "error_code retry_exhausted")
|
|
registry = json.loads(ops("failed").stdout)
|
|
check(any(r["attachment_id"] == att["id"] and r["attempt"] == 5 for r in registry), "failed registry entry")
|
|
ctx.note("failed after 5 attempts, registry entry present")
|
|
out = ops("replay", att["id"], "--actor", "teacher-demo")
|
|
ctx.note(out.stdout.strip())
|
|
ctx.wait(att["id"])
|
|
info = check_invariants(ctx, att["id"])
|
|
refused = ops("replay", att["id"], "--actor", "teacher-demo", check_rc=False)
|
|
check(refused.returncode == 3, "replay of a ready job must be refused")
|
|
info2 = inspect(att["id"])
|
|
check(len(events(info2, "claimed")) == len(events(info, "claimed")), "refused replay ran nothing")
|
|
check(info2["jobs"][0]["attempt"] == 6 and info2["jobs"][0]["replays"] == 1, "attempt 6 after replay")
|
|
ctx.note("replay of ready: %s" % refused.stdout.strip())
|
|
|
|
|
|
@scenario
|
|
def long_running_not_hung(ctx):
|
|
"""a 30 s step longer than the 20 s lease: heartbeat renews the lease -> no second attempt"""
|
|
arm("worker.before_transform", "delay:30")
|
|
att = ctx.upload(name="slow.png")
|
|
ctx.wait(att["id"], timeout=90)
|
|
info = check_invariants(ctx, att["id"])
|
|
check(not events(info, "lease_expired") and info["jobs"][0]["attempt"] == 1, "slow worker must keep its lease")
|
|
ctx.note("30 s processing with 20 s lease: attempt=1, no lease_expired")
|
|
|
|
|
|
# ============================================================ track 5.2: duplicates, reorder, versions
|
|
@scenario
|
|
def duplicate(ctx):
|
|
"""3 raw duplicates of an already completed job: ack as no-op, nothing rewritten"""
|
|
att = ctx.upload(name="dup.png")
|
|
ctx.wait(att["id"])
|
|
before = inspect(att["id"])
|
|
for _ in range(3):
|
|
ops("republish", att["id"], "--attempt", "1")
|
|
after = wait_for(lambda: (lambda i: i if len(events(i, "duplicate_noop")) >= 3 else None)(inspect(att["id"])), 30,
|
|
"3 duplicate no-ops")
|
|
check(after["variants"] == before["variants"] and after["attachment"] == before["attachment"], "state changed")
|
|
check_invariants(ctx, att["id"])
|
|
ctx.note("3 duplicates -> %d duplicate_noop, updated_at unchanged" % len(events(after, "duplicate_noop")))
|
|
|
|
|
|
@scenario
|
|
def reorder(ctx):
|
|
"""relay stopped; deliveries arrive out of order: attempt 1, then 1 again and 4, then the outbox backlog"""
|
|
dc("stop", "relay") # the outbox keeps its rows; we deliver raw messages in our own order
|
|
arm("worker.before_transform", "transient")
|
|
att = ctx.upload(name="reorder.png")
|
|
ops("republish", att["id"], "--attempt", "1") # attempt 1 runs and fails transiently -> attempt 2 in outbox
|
|
wait_for(lambda: events(inspect(att["id"]), "retry_scheduled"), 30, "first transient failure")
|
|
ops("republish", att["id"], "--attempt", "1") # stale: attempt 1 already happened
|
|
ops("republish", att["id"], "--attempt", "4") # "future": not the next attempt
|
|
wait_for(lambda: len(events(inspect(att["id"]), "stale_attempt_noop")) >= 2, 30, "stale no-ops")
|
|
check(ctx.get(att["id"])["status"] == "queued", "stale messages must not start an attempt")
|
|
dc("up", "-d", "--wait", "--no-build", "relay") # backlog: original attempt-1 row (stale now) + attempt 2
|
|
ctx.wait(att["id"])
|
|
info = check_invariants(ctx, att["id"])
|
|
check(info["jobs"][0]["attempt"] == 2 and len(events(info, "claimed")) == 2, "exactly two real attempts")
|
|
check(len(events(info, "stale_attempt_noop")) >= 3, "late outbox copy of attempt 1 must be stale too")
|
|
ctx.note("events: %s" % [e["event"] for e in info["events"]])
|
|
|
|
|
|
@scenario
|
|
def new_version(ctx):
|
|
"""v1 ready -> reprocess v2 -> v2 published; replaying a v1 message afterwards changes nothing"""
|
|
att = ctx.upload(name="versions.png")
|
|
v1 = ctx.wait(att["id"])
|
|
ops("reprocess", att["id"], "--pipeline-version", "2")
|
|
wait_for(lambda: inspect(att["id"])["attachment"]["pipeline_version"] == 2, 60, "v2 published")
|
|
v2 = ctx.get(att["id"])
|
|
check(v2["status"] == "ready", "ready never goes back to processing during reprocess")
|
|
opt1 = [v for v in v1["variants"] if v["name"] == "optimized"][0]["sha256"]
|
|
opt2 = [v for v in v2["variants"] if v["name"] == "optimized"][0]["sha256"]
|
|
check(opt1 != opt2, "v2 must publish different optimized bytes")
|
|
ops("republish", att["id"], "--pipeline-version", "1", "--attempt", "1")
|
|
info = wait_for(lambda: (lambda i: i if events(i, "duplicate_noop") else None)(inspect(att["id"])), 30, "v1 no-op")
|
|
check_invariants(ctx, att["id"], expect_version=2)
|
|
ctx.note("v1 optimized %s.. -> v2 %s..; stale v1 message: duplicate_noop" % (opt1[:12], opt2[:12]))
|
|
refused = ops("reprocess", att["id"], "--pipeline-version", "1", check_rc=False)
|
|
check(refused.returncode == 3, "downgrade must be refused")
|
|
|
|
|
|
@scenario
|
|
def stale_version_race(ctx):
|
|
"""v1 attempt is slow; v2 commits first; v1 then tries to commit -> superseded, v2 result kept"""
|
|
arm("worker.after_s3_before_commit", "delay:8")
|
|
att = ctx.upload(name="race.png")
|
|
wait_for(lambda: events(inspect(att["id"]), "fault_injected"), 30, "v1 inside the delay window")
|
|
ops("reprocess", att["id"], "--pipeline-version", "2")
|
|
info = wait_for(lambda: (lambda i: i if any(j["status"] == "superseded" for j in i["jobs"]) else None)(inspect(att["id"])),
|
|
60, "v1 superseded")
|
|
v1_keys = ["attachments/%s/v1/%s" % (uuid.UUID(att["id"]).hex, n) for n in ("crop.jpg", "optimized.webp", "thumbnail.jpg")]
|
|
check_invariants(ctx, att["id"], expect_version=2, leftovers=v1_keys)
|
|
check(all(v["pipeline_version"] == 2 for v in info["variants"]), "no v1 rows committed")
|
|
order = [(e["event"], e["detail"]) for e in info["events"] if e["event"] in ("ready", "superseded")]
|
|
ctx.note("commit order: %s; v1 objects are unreferenced leftovers for the sweeper" % order)
|
|
|
|
|
|
@scenario
|
|
def poison_message(ctx):
|
|
"""a message that violates jobs.schema.json goes to the DLQ instead of looping"""
|
|
before = stats()["dlq_messages"]
|
|
code = ("import asyncio,aio_pika;from app import config\n"
|
|
"async def m():\n"
|
|
" c=await aio_pika.connect(config.BROKER_URL);ch=await c.channel(publisher_confirms=True)\n"
|
|
" ex=await ch.get_exchange(config.BROKER_EXCHANGE)\n"
|
|
" await ex.publish(aio_pika.Message(b'{\"job_id\":\"x\",\"url\":\"http://evil\"}',delivery_mode=2),routing_key=config.BROKER_ROUTING_KEY)\n"
|
|
" await c.close()\n"
|
|
"asyncio.run(m())")
|
|
dc("exec", "-T", "ops", "python", "-c", code)
|
|
after = wait_for(lambda: (lambda s: s if s["dlq_messages"] == before + 1 else None)(stats()), 30, "DLQ")
|
|
ctx.note("DLQ %d -> %d" % (before, after["dlq_messages"]))
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument("--only", default="")
|
|
p.add_argument("--output", default=None)
|
|
args = arguments("Fault injection for the attachment pipeline", configure)
|
|
c = Client(args.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
ctx = Ctx(c, alice, bob, chat)
|
|
ops("fault", "clear")
|
|
only = {x for x in args.only.split(",") if x}
|
|
report = []
|
|
for fn in SCENARIOS:
|
|
if only and fn.__name__ not in only:
|
|
continue
|
|
print("--- %s: %s" % (fn.__name__, fn.__doc__.strip()))
|
|
ctx.notes = []
|
|
started = time.monotonic()
|
|
try:
|
|
fn(ctx)
|
|
finally:
|
|
ops("fault", "clear")
|
|
dc("up", "-d", "--wait", "--no-build", "worker", "relay", "api", "lb", "rabbitmq")
|
|
took = time.monotonic() - started
|
|
print("PASS %s (%.1fs)" % (fn.__name__, took))
|
|
report.append({"scenario": fn.__name__, "what": fn.__doc__.strip(), "seconds": round(took, 1), "observed": ctx.notes})
|
|
final = stats()
|
|
check(final["outbox_unpublished"] == 0 and final["jobs"].get("processing", 0) == 0, "global I5: %s" % final)
|
|
print("PASS: %d fault scenarios; final %s" % (len(report), json.dumps(final)))
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as fh:
|
|
json.dump({"scenarios": report, "final_stats": final}, fh, ensure_ascii=False, indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|