Files
mtusi2026lab6/scripts/verify_pipeline.py

514 lines
19 KiB
Python

#!/usr/bin/env python3
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
import argparse
import hashlib
import io
import json
import subprocess
import sys
import time
from pathlib import Path
from uuid import uuid4
from botocore.exceptions import ClientError
from cluster import ROOT, compose, inspect, instances, wait_lb
from PIL import Image
from pipeline_lab import settings
from messenger.broker import STREAM, Broker, Job
from messenger.pipeline import Pipeline
from messenger.repository import PostgresRepository, stamp
from messenger.storage import ObjectStore
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check # noqa: E402
def until(fn, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
result = fn()
if result:
return result
time.sleep(0.2)
raise AssertionError("Timed out waiting for a durable pipeline condition")
def main():
parser = argparse.ArgumentParser(__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8086")
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
args = parser.parse_args()
if args.output.exists():
parser.error("Output already exists; choose a new evidence file")
config = settings()
db = PostgresRepository(config.database_url, pool_size=4)
db.open()
broker, store = Broker(config), ObjectStore(config)
c = Client(args.base_url)
alice, bob, eve, room = c.bootstrap()
records, accepted = [], []
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
def query(sql, params=()):
with db.connection() as conn:
return conn.execute(sql, params).fetchall()
def job(a, version=1):
return query(
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
(a["id"], version),
)[0]
def audit(j):
return query(
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
(j["job_id"],),
)
def injected(j, phase):
return any(
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
)
def fault(j, phase, action, count=1):
with db.connection(write=True) as conn:
conn.execute(
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
(j["job_id"], phase, action, count),
)
def upload():
a, original = c.attachment(alice, room)
check(original == data, "Unexpected fixture")
accepted.append(a["id"])
return a
def ready(a):
value = c.wait_ready(bob, a)
path = "/api/v1/attachments/" + a["id"]
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
for v in value["variants"]:
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
with Image.open(io.BytesIO(content)) as im:
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
check(len(value["variants"]) == 3, "Missing variants")
check(
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
"Unexpected duplicate link",
)
return value
def saved(name, a, start, **details):
j = job(a, details.get("version", 1))
current = query(
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
)[0]
keys = [
o["Key"]
for o in store.client.list_objects_v2(
Bucket=store.bucket,
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
).get("Contents", [])
]
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
record = {
"scenario": name,
"attachment_id": a["id"],
"job_id": str(j["job_id"]),
"elapsed_seconds": time.monotonic() - start,
"final_state": current["storage_state"],
"attempts": j["attempt"],
"final_version": current["pipeline_version"],
"published_objects": len(keys),
"audit": audit(j),
**details,
}
records.append(record)
print("PASS: " + name, flush=True)
def envelope(j, attempt=None):
return Job(
schema_version=1,
type="attachment.process.v1",
job_id=str(j["job_id"]),
attachment_id=str(j["attachment_id"]),
pipeline_version=j["pipeline_version"],
attempt=attempt or max(1, j["attempt"]),
created_at=stamp(j["created_at"]),
)
try:
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
check(
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
is None,
"Gap was not established",
)
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
before = instances()
compose("restart", "--timeout", "3", "api")
wait_lb(args.base_url)
after = instances()
check(
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
"API did not restart",
)
compose("start", "relay")
until(
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
"published_at"
]
)
entry_ids = [
id
for id, fields in broker.client.xrange(STREAM)
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
]
check(bool(entry_ids), "Broker has no accepted work")
broker_id = compose("ps", "-q", "broker").strip()
broker_before = inspect(broker_id)
subprocess.run(
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
)
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
def broker_restored():
try:
return broker.client.ping()
except Exception:
return False
until(broker_restored)
check(
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
"Durable stream lost confirmed work",
)
compose("start", "worker")
ready(a)
history = c.call(
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
)["json"]
check(message in history["items"], "Queued attachment message lost")
saved(
"commit_publish_gap_api_restart_broker_sigkill",
a,
start,
api_before=before,
api_after=after,
broker_before=broker_before,
broker_after=inspect(broker_id),
broker_entry_ids=entry_ids,
message_id=message["id"],
)
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "after_publish", "crash")
compose("start", "relay")
until(lambda: injected(j, "after_publish"))
until(
lambda: (
sum(
json.loads(f["job"])["job_id"] == str(j["job_id"])
for _, f in broker.client.xrange(STREAM)
)
>= 2
)
)
compose("start", "worker")
ready(a)
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
saved("publish_before_outbox_commit_duplicate", a, start)
for phase in ("before_effect", "after_effect", "after_commit"):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, phase, "crash")
compose("start", "worker")
ready(a)
until(lambda j=j, phase=phase: injected(j, phase))
if phase == "after_commit":
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
check(job(a)["attempt"] == 1, "Committed job was processed twice")
else:
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
saved("worker_crash_" + phase, a, start)
for action, count, expected in (
("transient", 2, "ready"),
("hang", 1, "ready"),
("permanent", 1, "failed"),
("transient", 5, "failed"),
):
compose("stop", "--timeout", "2", "worker")
start = time.monotonic()
a = upload()
j = job(a)
fault(j, "before_effect", action, count)
compose("start", "worker")
path = "/api/v1/attachments/" + a["id"]
if expected == "failed":
until(
lambda path=path: (
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
== "failed"
)
)
terminal = job(a)
check(terminal["attempt"] == count, "Wrong retry budget")
c.call("GET", path + "/content", 409, actor=bob)
c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/messages",
409,
actor=alice,
body={"text": "", "attachment_ids": [a["id"]]},
headers={"Idempotency-Key": uuid4().hex},
)
# Explicit CLI replay, same logical job; no public administrator endpoint.
replay = json.loads(
compose(
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
)
)
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
else:
terminal = None
ready(a)
events = audit(j)
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
rejected = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"worker",
"python",
"-m",
"messenger.operator",
"replay",
a["id"],
],
cwd=ROOT,
capture_output=True,
)
check(rejected.returncode != 0, "Ready replay was accepted")
saved(
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
a,
start,
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
if terminal
else None,
backoff_delays=delays,
)
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
compose("stop", "--timeout", "2", "worker", "relay")
start = time.monotonic()
a = upload()
old = job(a)
old_envelope = envelope(old, 1)
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
check(outcome == "claimed", "Cannot establish old writer")
Pipeline(db, config).replay(a["id"], new_version=True)
new = job(a, 2)
# Publish v2 first, then v1, plus duplicates through both worker slots.
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
broker.publish(value)
compose("start", "worker", "relay")
current = ready(a)
row = db.attachment(a["id"])
check(
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
"Old writer overwrote v2",
)
for _ in range(20):
broker.publish(old_envelope)
broker.publish(envelope(new, 1))
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
check(
db.attachment(a["id"])["object_key"] == row["object_key"],
"Stale delivery changed current pointer",
)
check(
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
== current,
"Old version mutated ready metadata",
)
saved(
"reorder_stale_v1_after_v2_and_publication_fence",
a,
start,
version=2,
old_audit=audit(old),
duplicates=44,
)
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
a = {"id": accepted[0]}
for suffix in ("", "/content", "/content?variant=crop"):
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
# Direct bucket access and write/delete privilege checks use no public credentials.
import boto3
from botocore import UNSIGNED
from botocore.config import Config
first = db.attachment(accepted[0])
unsigned = boto3.client(
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
)
try:
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
raise AssertionError("Anonymous S3 read succeeded")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
)
finally:
unsigned.close()
denied = []
for role, action in (
("api", "put_object"),
("api", "delete_object"),
("worker", "delete_object"),
("reconciler", "put_object"),
):
scoped = ObjectStore(settings(role))
try:
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
if action == "put_object":
kw["Body"] = b"forbidden"
try:
getattr(scoped.client, action)(**kw)
raise AssertionError("IAM action unexpectedly allowed")
except ClientError as exc:
check(
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
"Wrong IAM status",
)
denied.append(role + ":" + action)
finally:
scoped.close()
# Stop the actual uploader, then read original and thumbnail through another replica.
response = c.call(
"POST",
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
202,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
cross = response["json"]
accepted.append(cross["id"])
ready(cross)
uploader = response["headers"]["X-Instance-Id"]
process = next(v for v in instances() if v["instance"] == uploader)
subprocess.run(
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
)
try:
wait_lb(args.base_url, count=1)
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
check(
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
"Other replica could not serve the original",
)
c.call(
"GET",
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
actor=bob,
)
cross_proof = {
"uploader": uploader,
"stopped": inspect(process["id"]),
"reader": download["headers"]["X-Instance-Id"],
}
finally:
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
wait_lb(args.base_url)
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
check(
not any(
actor["token"] in logs or actor["registration"]["password"] in logs
for actor in (alice, bob, eve)
),
"Credential in logs",
)
states = query(
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
)
check(
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
"Confirmed work lost",
)
stats = compose(
"exec",
"-T",
"worker",
"python",
"-c",
"import json;from pathlib import Path;print(json.dumps({"
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
)
result = {
"anonymous_s3_status": 403,
"denied_iam_actions": denied,
"cross_replica": cross_proof,
"track": 2,
"delivery": "at-least-once",
"effect_key": ["attachment_id", "pipeline_version", "variant"],
"worker_concurrency": config.worker_concurrency,
"prefetch_per_slot": 1,
"job_timeout_seconds": config.job_timeout,
"lease_seconds": config.lease_seconds,
"max_attempts": config.job_max_attempts,
"scenarios": records,
"accepted": len(accepted),
"lost_confirmed": 0,
"stale_overwrites": 0,
"worker_cgroup": json.loads(stats),
"logs_credentials_absent": True,
}
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
finally:
compose("start", "worker", "relay", "broker", "api")
broker.close()
store.close()
db.close()
if __name__ == "__main__":
main()