Files
mtusi2026lab5/scripts/verify_pipeline.py
T

605 lines
24 KiB
Python

"""Repeatable fault injection for grade 5, track 2; synthetic data only."""
import argparse
import hashlib
import io
import json
import os
import time
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from uuid import uuid4
import httpx
from PIL import Image
os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml")
os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
os.environ.setdefault("no_proxy", "localhost,127.0.0.1")
from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready
PREFIX = """import io,json,hashlib,time,urllib.request
from uuid import UUID,uuid4
from psycopg.types.json import Jsonb
from src.pipeline import database,broker,envelope,STREAM,GROUP
from src.object_store import ObjectStore,BUCKET
from src.operator import change
from src.reconcile import reconcile
db,store,queue=database(),ObjectStore(),broker()
"""
def inside(code):
cid = compose("ps", "-q", "controller").split()[0]
return json.loads(
command(
["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code
)
)
def wait(fn, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = fn()
if value:
return value
time.sleep(0.15)
raise TimeoutError("Fault scenario condition not reached")
def stop(role):
compose("stop", "--timeout", "1", role)
def start(role):
compose("up", "-d", "--no-deps", "--no-recreate", role)
def row(job):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))"
)
def inject(job, phase, action, remaining=1):
return inside(
f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))"
)
def hit(identity):
return inside(
f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))"
)
def operator(job, version=None):
return inside(
f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))"
)
def duplicate(job, count=5):
return inside(
f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))"
)
def inspect_effect(job):
return inside(f"""with db.connection() as c:
j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()
v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall()
p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n']
e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall()
keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")]
print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--output", default="evidence/pipeline-repeat.json")
parser.add_argument("--repeats", type=int, default=2)
args = parser.parse_args()
output = ROOT / args.output
if output.exists():
raise ValueError("Choose a new output path")
report = {
"source_commit": command(["git", "rev-parse", "HEAD"]),
"date": datetime.now(timezone.utc).isoformat(),
"track": 2,
"complete": False,
"cases": [],
}
output.parent.mkdir(parents=True, exist_ok=True)
def save(name, details):
report["cases"].append({"case": name, **details})
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS:", name, flush=True)
try:
ready()
# Check instrumentation is really enabled and production default is off.
cid = compose("ps", "-q", "api").split()[0]
assert (
command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import os;print(os.getenv('PIPELINE_FAULTS'))",
]
)
== "1"
)
stop("reconciler")
with httpx.Client(
base_url=BASE, trust_env=False, timeout=25
) as client:
user, _, session = actor(client)
h = auth(session)
c = client.post(
"/api/v1/chats",
headers=h,
json={
"title": "Pipeline synthetic faults",
"member_ids": [user["id"]],
},
)
assert c.status_code == 201
chat = c.json()["id"]
raw_io = io.BytesIO()
Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save(
raw_io, format="PNG"
)
raw = raw_io.getvalue()
def upload():
r = client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "synthetic.png"},
content=raw,
)
assert r.status_code == 202, r.text
a = r.json()
assert a["status"] == "queued" and not a["variants"]
j = inside(
f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))"
)
return a, j
def metadata(a):
r = client.get("/api/v1/attachments/" + a["id"], headers=h)
assert r.status_code == 200, r.text
return r.json()
def until(a, state="ready", timeout=90):
return wait(
lambda: (
r if (r := metadata(a))["status"] == state else None
),
timeout,
)
def confirm(a, j):
until(a)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.status_code == 200 and downloaded.content == raw
)
m = metadata(a)
assert (
m["sha256"] == hashlib.sha256(raw).hexdigest()
and m["error_code"] is None
)
assert {v["name"] for v in m["variants"]} == {
"thumbnail",
"crop",
"optimized",
}
for v in m["variants"]:
r = client.get(
f"/api/v1/attachments/{a['id']}/content",
headers=h,
params={"variant": v["name"]},
)
assert (
r.status_code == 200
and hashlib.sha256(r.content).hexdigest()
== v["sha256"]
)
with Image.open(io.BytesIO(r.content)) as image:
assert (
image.size
== {
"thumbnail": (256, 160),
"crop": (128, 128),
"optimized": (320, 200),
}[v["name"]]
)
assert not image.getexif()
effect = inspect_effect(j)
assert (
effect["publications"] == 1
and len(effect["variants"]) == 4
and len(effect["physical_keys"]) == 4
)
return effect
# Source-only crash before the DB handoff must never be reported as accepted.
stop("worker")
stop("relay")
marker = inject(chat, "api_after_staging", "wait")
before = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(
lambda: client.post(
f"/api/v1/chats/{chat}/attachments",
headers={**h, "Content-Type": "image/png"},
params={"filename": "unconfirmed.png"},
content=raw,
)
)
wait(lambda marker=marker: hit(marker))
compose("kill", "-s", "SIGKILL", "api")
try:
response = future.result(15)
assert response.status_code != 202
except httpx.HTTPError:
pass
pool.shutdown()
start("api")
ready()
after = inside(
f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))"
)
assert before == after
save(
"staging_before_handoff_crash",
{
"no_202": True,
"no_attachment_or_job_created": True,
"orphan_retained_for_cleanup": True,
},
)
# API durable commit/outbox survives restart without a broker or worker.
a, j = upload()
linked = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert linked.status_code == 201
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
compose("restart", "api")
ready()
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
start("relay")
start("worker")
effect = confirm(a, j)
save(
"commit_before_publish_and_worker_stopped",
{
"attachment": a["id"],
"effect": effect,
"queued_link_allowed": True,
"api_and_broker_restarted": True,
},
)
# Broker-volume survival with a confirmed, still-unconsumed stream entry.
stop("worker")
a, j = upload()
wait(
lambda: inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))"
)
)
stop("relay")
ids = inside("print(json.dumps(queue.xrange(STREAM)))")
compose("kill", "-s", "SIGKILL", "broker")
start("broker")
wait(lambda: inside("print(json.dumps(queue.ping()))"))
restored = inside("print(json.dumps(queue.xrange(STREAM)))")
assert restored == ids, {"before": ids, "after": restored}
start("relay")
start("worker")
save(
"confirmed_broker_entry_survives_sigkill",
{
"effect": confirm(a, j),
"same_stream_entries": True,
"entry_count": len(ids),
},
)
# Confirm succeeded, PG marker did not: relay recovery may publish a duplicate.
stop("worker")
stop("relay")
a, j = upload()
marker = inject(j, "relay_after_publish", "crash")
start("relay")
wait(lambda marker=marker: hit(marker))
time.sleep(0.5)
unpublished = inside(
f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))"
)
assert unpublished >= 1
start("relay")
start("worker")
save(
"relay_crash_after_xadd_before_pg_marker",
{
"unpublished_after_crash": unpublished,
"effect": confirm(a, j),
},
)
# Repeated before/after effect and commit/ack windows.
for repeat in range(args.repeats):
for phase in ["before_effect", "after_effect", "after_commit"]:
stop("worker")
a, j = upload()
marker = inject(j, phase, "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
at_crash = inspect_effect(j)
assert at_crash["publications"] == (
1 if phase == "after_commit" else 0
)
assert len(at_crash["physical_keys"]) == (
0 if phase == "before_effect" else 4
)
compose("kill", "-s", "SIGKILL", "worker")
start("worker")
effect = confirm(a, j)
save(
f"{phase}_sigkill_repeat_{repeat + 1}",
{"at_crash": at_crash, "effect": effect},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient")
start("worker")
effect = confirm(a, j)
assert effect["job"]["attempt"] == 2
save("transient_retry_with_jitter", {"effect": effect})
stop("worker")
a, j = upload()
inject(j, "before_effect", "hang")
start("worker")
effect = confirm(a, j)
assert any(
e["event"] == "retry"
and e["detail"]["code"] == "worker_timeout"
for e in effect["events"]
)
save("hung_child_killed_and_retried", {"effect": effect})
stop("worker")
a, j = upload()
inject(j, "before_effect", "permanent")
start("worker")
failed = until(a, "failed")
assert failed["error_code"] == "invalid_image"
assert (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 409
)
r = client.post(
f"/api/v1/chats/{chat}/messages",
headers={**h, "Idempotency-Key": uuid4().hex},
json={"text": "", "attachment_ids": [a["id"]]},
)
assert r.status_code == 409
assert operator(j) == j
effect = confirm(a, j)
assert effect["job"]["cycle"] == 2
try:
operator(j)
except RuntimeError:
pass
else:
raise AssertionError("Replay of ready must be rejected")
save(
"permanent_failed_registry_and_audited_replay",
{
"failed": failed,
"effect": effect,
"ready_replay_rejected": True,
},
)
stop("worker")
a, j = upload()
inject(j, "before_effect", "transient", remaining=5)
start("worker")
failed = until(a, "failed", timeout=150)
assert (
failed["error_code"] == "retry_exhausted"
and row(j)["attempt"] == 5
)
operator(j)
save(
"finite_retry_exhaustion_then_explicit_replay",
{"failed": failed, "effect": confirm(a, j)},
)
# Reverse notification order and add concurrent copies: one publication/key.
stop("worker")
stop("relay")
a, j = upload()
b, k = upload()
duplicate(k, 8)
duplicate(j, 8)
start("worker")
start("relay")
ea = confirm(a, j)
confirm(b, k)
duplicate(j, 10)
duplicate(k, 10)
wait(
lambda: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"]
save(
"reordered_and_duplicate_notifications",
{
"effects": [inspect_effect(j), inspect_effect(k)],
"added_duplicates": 36,
"no_extra_attempt_after_ready": True,
},
)
# Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS.
for repeat in range(args.repeats):
stop("worker")
a, j = upload()
marker = inject(j, "after_effect", "wait")
start("worker")
wait(lambda marker=marker: hit(marker))
k = operator(j, version=2)
v2 = confirm(a, k)
hashes = {v["name"]: v["sha256"] for v in v2["variants"]}
inside(
f"with db.connection() as c: c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))\nprint('true')"
)
wait(
lambda j=j: any(
e["event"] == "stale_completion_rejected"
for e in inspect_effect(j)["events"]
)
)
duplicate(j, 12)
wait(
lambda j=j: any(
e["event"] == "duplicate_decision"
for e in inspect_effect(j)["events"]
)
)
assert hashes == {
v["name"]: v["sha256"] for v in confirm(a, k)["variants"]
}
assert inspect_effect(j)["publications"] == 0
save(
f"stale_v1_after_v2_repeat_{repeat + 1}",
{
"v1": inspect_effect(j),
"v2": inspect_effect(k),
"new_hashes_unchanged": True,
},
)
# No arbitrary URL/bytes fields in queue envelopes.
invalid = inside(
"bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')"
)
assert invalid
wait(
lambda: inside(
"with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))"
)
)
save(
"malformed_notification_rejected", {"durable_diagnostic": True}
)
# Private bucket, bounded IAM, and download after uploader process dies.
a, j = upload()
confirm(a, j)
key = f"objects/{a['id']}/v1/original"
security = inside(f"""from minio.error import S3Error
from urllib.error import HTTPError
try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3)
except HTTPError as e: anonymous=e.code
else: raise AssertionError('Anonymous access allowed')
denials={{}}
for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]:
try: fn()
except S3Error as e: denials[name]=e.code
else: raise AssertionError('IAM allowed forbidden action')
print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""")
assert security["anonymous"] == 403 and set(
security["iam"].values()
) == {"AccessDenied"}
selected = None
for cid in compose("ps", "-q", "api").split():
alias = command(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
]
)
uploader = inside(
f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))"
)
if alias == uploader:
selected = cid
assert selected
command(["docker", "stop", "--time", "1", selected])
ready()
wait(
lambda: (
client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
).status_code
== 200
)
)
downloaded = client.get(
f"/api/v1/attachments/{a['id']}/content", headers=h
)
assert (
downloaded.content == raw
and downloaded.headers["X-Instance-Id"] != uploader
)
start("api")
ready()
save(
"private_iam_and_other_api_download",
{
"security": security,
"attachment": a["id"],
"uploader": uploader,
"downloader": downloaded.headers["X-Instance-Id"],
"sha256": hashlib.sha256(raw).hexdigest(),
},
)
report["worker_metrics"] = inside(
"with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))"
)
report["audit"] = inside(
"with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))"
)
report["complete"] = True
finally:
for role in ["api", "broker", "relay", "worker", "reconciler"]:
try:
start(role)
except Exception:
pass
output.write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
)
print("PASS: complete versioned pipeline fault matrix", flush=True)
if __name__ == "__main__":
main()