forked from mtusi2026/mtusi2026lab5
514 lines
19 KiB
Python
514 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
"""Real-process fault matrix for the versioned pipeline; test Compose override required."""
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
from botocore.exceptions import ClientError
|
|
from cluster import ROOT, compose, inspect, instances, wait_lb
|
|
from PIL import Image
|
|
from pipeline_lab import settings
|
|
|
|
from messenger.broker import STREAM, Broker, Job
|
|
from messenger.pipeline import Pipeline
|
|
from messenger.repository import PostgresRepository, stamp
|
|
from messenger.storage import ObjectStore
|
|
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, check # noqa: E402
|
|
|
|
|
|
def until(fn, timeout=60):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
result = fn()
|
|
if result:
|
|
return result
|
|
time.sleep(0.2)
|
|
raise AssertionError("Timed out waiting for a durable pipeline condition")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(__doc__)
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8085")
|
|
parser.add_argument("--output", type=Path, default=ROOT / "evidence/pipeline.json")
|
|
args = parser.parse_args()
|
|
if args.output.exists():
|
|
parser.error("Output already exists; choose a new evidence file")
|
|
config = settings()
|
|
db = PostgresRepository(config.database_url, pool_size=4)
|
|
db.open()
|
|
broker, store = Broker(config), ObjectStore(config)
|
|
c = Client(args.base_url)
|
|
alice, bob, eve, room = c.bootstrap()
|
|
records, accepted = [], []
|
|
data = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
def query(sql, params=()):
|
|
with db.connection() as conn:
|
|
return conn.execute(sql, params).fetchall()
|
|
|
|
def job(a, version=1):
|
|
return query(
|
|
"SELECT * FROM attachment_jobs WHERE attachment_id=%s AND pipeline_version=%s",
|
|
(a["id"], version),
|
|
)[0]
|
|
|
|
def audit(j):
|
|
return query(
|
|
"SELECT event,details,created_at FROM pipeline_audit WHERE job_id=%s ORDER BY sequence",
|
|
(j["job_id"],),
|
|
)
|
|
|
|
def injected(j, phase):
|
|
return any(
|
|
e["event"] == "fault_injected" and e["details"]["phase"] == phase for e in audit(j)
|
|
)
|
|
|
|
def fault(j, phase, action, count=1):
|
|
with db.connection(write=True) as conn:
|
|
conn.execute(
|
|
"INSERT INTO pipeline_faults(job_id,phase,action,remaining) VALUES (%s,%s,%s,%s)",
|
|
(j["job_id"], phase, action, count),
|
|
)
|
|
|
|
def upload():
|
|
a, original = c.attachment(alice, room)
|
|
check(original == data, "Unexpected fixture")
|
|
accepted.append(a["id"])
|
|
return a
|
|
|
|
def ready(a):
|
|
value = c.wait_ready(bob, a)
|
|
path = "/api/v1/attachments/" + a["id"]
|
|
check(c.call("GET", path + "/content", actor=bob)["bytes"] == data, "Original changed")
|
|
for v in value["variants"]:
|
|
content = c.call("GET", path + "/content?variant=" + v["name"], actor=bob)["bytes"]
|
|
check(hashlib.sha256(content).hexdigest() == v["sha256"], "Variant checksum mismatch")
|
|
with Image.open(io.BytesIO(content)) as im:
|
|
check(im.size == (v["width"], v["height"]), "Variant dimensions mismatch")
|
|
check(len(value["variants"]) == 3, "Missing variants")
|
|
check(
|
|
len(query("SELECT 1 FROM message_attachments WHERE attachment_id=%s", (a["id"],))) <= 1,
|
|
"Unexpected duplicate link",
|
|
)
|
|
return value
|
|
|
|
def saved(name, a, start, **details):
|
|
j = job(a, details.get("version", 1))
|
|
current = query(
|
|
"SELECT storage_state,pipeline_version FROM attachments WHERE id=%s", (a["id"],)
|
|
)[0]
|
|
keys = [
|
|
o["Key"]
|
|
for o in store.client.list_objects_v2(
|
|
Bucket=store.bucket,
|
|
Prefix="objects/" + a["id"] + "/v" + str(current["pipeline_version"]) + "/",
|
|
).get("Contents", [])
|
|
]
|
|
check(len(keys) == 4, "Repeated deliveries multiplied the published variant set")
|
|
record = {
|
|
"scenario": name,
|
|
"attachment_id": a["id"],
|
|
"job_id": str(j["job_id"]),
|
|
"elapsed_seconds": time.monotonic() - start,
|
|
"final_state": current["storage_state"],
|
|
"attempts": j["attempt"],
|
|
"final_version": current["pipeline_version"],
|
|
"published_objects": len(keys),
|
|
"audit": audit(j),
|
|
**details,
|
|
}
|
|
records.append(record)
|
|
print("PASS: " + name, flush=True)
|
|
|
|
def envelope(j, attempt=None):
|
|
return Job(
|
|
schema_version=1,
|
|
type="attachment.process.v1",
|
|
job_id=str(j["job_id"]),
|
|
attachment_id=str(j["attachment_id"]),
|
|
pipeline_version=j["pipeline_version"],
|
|
attempt=attempt or max(1, j["attempt"]),
|
|
created_at=stamp(j["created_at"]),
|
|
)
|
|
|
|
try:
|
|
# Actual COMMIT/publish gap: no relay exists while the acknowledged upload is stored.
|
|
compose("stop", "--timeout", "2", "worker", "relay")
|
|
start = time.monotonic()
|
|
a = upload()
|
|
j = job(a)
|
|
check(
|
|
query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
|
"published_at"
|
|
]
|
|
is None,
|
|
"Gap was not established",
|
|
)
|
|
message = c.message(alice, room, "", attachments=[a["id"]])["json"]
|
|
c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", 409, actor=bob)
|
|
before = instances()
|
|
compose("restart", "--timeout", "3", "api")
|
|
wait_lb(args.base_url)
|
|
after = instances()
|
|
check(
|
|
{r["instance"] for r in before}.isdisjoint({r["instance"] for r in after}),
|
|
"API did not restart",
|
|
)
|
|
compose("start", "relay")
|
|
until(
|
|
lambda: query("SELECT published_at FROM job_outbox WHERE job_id=%s", (j["job_id"],))[0][
|
|
"published_at"
|
|
]
|
|
)
|
|
entry_ids = [
|
|
id
|
|
for id, fields in broker.client.xrange(STREAM)
|
|
if json.loads(fields["job"])["job_id"] == str(j["job_id"])
|
|
]
|
|
check(bool(entry_ids), "Broker has no accepted work")
|
|
broker_id = compose("ps", "-q", "broker").strip()
|
|
broker_before = inspect(broker_id)
|
|
subprocess.run(
|
|
["docker", "kill", "--signal", "KILL", broker_id], check=True, capture_output=True
|
|
)
|
|
subprocess.run(["docker", "start", broker_id], check=True, capture_output=True)
|
|
|
|
def broker_restored():
|
|
try:
|
|
return broker.client.ping()
|
|
except Exception:
|
|
return False
|
|
|
|
until(broker_restored)
|
|
check(
|
|
set(entry_ids) <= {id for id, _ in broker.client.xrange(STREAM)},
|
|
"Durable stream lost confirmed work",
|
|
)
|
|
compose("start", "worker")
|
|
ready(a)
|
|
history = c.call(
|
|
"GET", "/api/v1/chats/" + room["id"] + "/messages", actor=bob, schema="MessagePage"
|
|
)["json"]
|
|
check(message in history["items"], "Queued attachment message lost")
|
|
saved(
|
|
"commit_publish_gap_api_restart_broker_sigkill",
|
|
a,
|
|
start,
|
|
api_before=before,
|
|
api_after=after,
|
|
broker_before=broker_before,
|
|
broker_after=inspect(broker_id),
|
|
broker_entry_ids=entry_ids,
|
|
message_id=message["id"],
|
|
)
|
|
|
|
# A publisher can lose its COMMIT after XADD; same logical ID is sent twice.
|
|
compose("stop", "--timeout", "2", "worker", "relay")
|
|
start = time.monotonic()
|
|
a = upload()
|
|
j = job(a)
|
|
fault(j, "after_publish", "crash")
|
|
compose("start", "relay")
|
|
until(lambda: injected(j, "after_publish"))
|
|
until(
|
|
lambda: (
|
|
sum(
|
|
json.loads(f["job"])["job_id"] == str(j["job_id"])
|
|
for _, f in broker.client.xrange(STREAM)
|
|
)
|
|
>= 2
|
|
)
|
|
)
|
|
compose("start", "worker")
|
|
ready(a)
|
|
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
|
saved("publish_before_outbox_commit_duplicate", a, start)
|
|
|
|
for phase in ("before_effect", "after_effect", "after_commit"):
|
|
compose("stop", "--timeout", "2", "worker")
|
|
start = time.monotonic()
|
|
a = upload()
|
|
j = job(a)
|
|
fault(j, phase, "crash")
|
|
compose("start", "worker")
|
|
ready(a)
|
|
until(lambda j=j, phase=phase: injected(j, phase))
|
|
if phase == "after_commit":
|
|
until(lambda j=j: any(e["event"] == "duplicate_ready" for e in audit(j)))
|
|
check(job(a)["attempt"] == 1, "Committed job was processed twice")
|
|
else:
|
|
check(job(a)["attempt"] == 2, "Expired lease did not recover exactly once")
|
|
saved("worker_crash_" + phase, a, start)
|
|
|
|
for action, count, expected in (
|
|
("transient", 2, "ready"),
|
|
("hang", 1, "ready"),
|
|
("permanent", 1, "failed"),
|
|
("transient", 5, "failed"),
|
|
):
|
|
compose("stop", "--timeout", "2", "worker")
|
|
start = time.monotonic()
|
|
a = upload()
|
|
j = job(a)
|
|
fault(j, "before_effect", action, count)
|
|
compose("start", "worker")
|
|
path = "/api/v1/attachments/" + a["id"]
|
|
if expected == "failed":
|
|
until(
|
|
lambda path=path: (
|
|
c.call("GET", path, actor=bob, schema="Attachment")["json"]["status"]
|
|
== "failed"
|
|
)
|
|
)
|
|
terminal = job(a)
|
|
check(terminal["attempt"] == count, "Wrong retry budget")
|
|
c.call("GET", path + "/content", 409, actor=bob)
|
|
c.call(
|
|
"POST",
|
|
"/api/v1/chats/" + room["id"] + "/messages",
|
|
409,
|
|
actor=alice,
|
|
body={"text": "", "attachment_ids": [a["id"]]},
|
|
headers={"Idempotency-Key": uuid4().hex},
|
|
)
|
|
# Explicit CLI replay, same logical job; no public administrator endpoint.
|
|
replay = json.loads(
|
|
compose(
|
|
"exec",
|
|
"-T",
|
|
"worker",
|
|
"python",
|
|
"-m",
|
|
"messenger.operator",
|
|
"replay",
|
|
a["id"],
|
|
)
|
|
)
|
|
check(replay["job_id"] == str(j["job_id"]), "Replay changed logical job")
|
|
else:
|
|
terminal = None
|
|
ready(a)
|
|
events = audit(j)
|
|
delays = [e["details"]["delay_seconds"] for e in events if e["event"] == "queued"]
|
|
check(all(0.25 <= delay <= 30 for delay in delays), "Invalid bounded jitter")
|
|
rejected = subprocess.run(
|
|
[
|
|
"docker",
|
|
"compose",
|
|
"--env-file",
|
|
".env",
|
|
"exec",
|
|
"-T",
|
|
"worker",
|
|
"python",
|
|
"-m",
|
|
"messenger.operator",
|
|
"replay",
|
|
a["id"],
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
)
|
|
check(rejected.returncode != 0, "Ready replay was accepted")
|
|
saved(
|
|
action + "_" + str(count) + ("_failed_then_explicit_replay" if terminal else ""),
|
|
a,
|
|
start,
|
|
terminal_before_replay={k: terminal[k] for k in ("state", "attempt", "error_code")}
|
|
if terminal
|
|
else None,
|
|
backoff_delays=delays,
|
|
)
|
|
|
|
# Reorder and stale v1 delivered after v2; exercise the DB publication fence directly too.
|
|
compose("stop", "--timeout", "2", "worker", "relay")
|
|
start = time.monotonic()
|
|
a = upload()
|
|
old = job(a)
|
|
old_envelope = envelope(old, 1)
|
|
outcome, old_claim = Pipeline(db, config).claim(old_envelope)
|
|
check(outcome == "claimed", "Cannot establish old writer")
|
|
Pipeline(db, config).replay(a["id"], new_version=True)
|
|
new = job(a, 2)
|
|
# Publish v2 first, then v1, plus duplicates through both worker slots.
|
|
for value in (envelope(new, 1), old_envelope, envelope(new, 1), old_envelope):
|
|
broker.publish(value)
|
|
compose("start", "worker", "relay")
|
|
current = ready(a)
|
|
row = db.attachment(a["id"])
|
|
check(
|
|
not Pipeline(db, config).publish(old_claim, "objects/old-writer", []),
|
|
"Old writer overwrote v2",
|
|
)
|
|
for _ in range(20):
|
|
broker.publish(old_envelope)
|
|
broker.publish(envelope(new, 1))
|
|
until(lambda: sum(e["event"] == "stale_delivery" for e in audit(old)) >= 21)
|
|
until(lambda: sum(e["event"] == "duplicate_ready" for e in audit(new)) >= 20)
|
|
check(
|
|
db.attachment(a["id"])["object_key"] == row["object_key"],
|
|
"Stale delivery changed current pointer",
|
|
)
|
|
check(
|
|
c.call("GET", "/api/v1/attachments/" + a["id"], actor=bob, schema="Attachment")["json"]
|
|
== current,
|
|
"Old version mutated ready metadata",
|
|
)
|
|
saved(
|
|
"reorder_stale_v1_after_v2_and_publication_fence",
|
|
a,
|
|
start,
|
|
version=2,
|
|
old_audit=audit(old),
|
|
duplicates=44,
|
|
)
|
|
|
|
# ACL remains enforced through both API replicas; S3 is private and role-scoped.
|
|
a = {"id": accepted[0]}
|
|
for suffix in ("", "/content", "/content?variant=crop"):
|
|
c.call("GET", "/api/v1/attachments/" + a["id"] + suffix, 404, actor=eve)
|
|
# Direct bucket access and write/delete privilege checks use no public credentials.
|
|
import boto3
|
|
from botocore import UNSIGNED
|
|
from botocore.config import Config
|
|
|
|
first = db.attachment(accepted[0])
|
|
unsigned = boto3.client(
|
|
"s3", endpoint_url=config.s3_endpoint, config=Config(signature_version=UNSIGNED)
|
|
)
|
|
try:
|
|
unsigned.get_object(Bucket=store.bucket, Key=first["object_key"])
|
|
raise AssertionError("Anonymous S3 read succeeded")
|
|
except ClientError as exc:
|
|
check(
|
|
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403, "Wrong anonymous status"
|
|
)
|
|
finally:
|
|
unsigned.close()
|
|
denied = []
|
|
for role, action in (
|
|
("api", "put_object"),
|
|
("api", "delete_object"),
|
|
("worker", "delete_object"),
|
|
("reconciler", "put_object"),
|
|
):
|
|
scoped = ObjectStore(settings(role))
|
|
try:
|
|
kw = {"Bucket": store.bucket, "Key": "objects/" + uuid4().hex + "/forbidden"}
|
|
if action == "put_object":
|
|
kw["Body"] = b"forbidden"
|
|
try:
|
|
getattr(scoped.client, action)(**kw)
|
|
raise AssertionError("IAM action unexpectedly allowed")
|
|
except ClientError as exc:
|
|
check(
|
|
exc.response["ResponseMetadata"]["HTTPStatusCode"] == 403,
|
|
"Wrong IAM status",
|
|
)
|
|
denied.append(role + ":" + action)
|
|
finally:
|
|
scoped.close()
|
|
# Stop the actual uploader, then read original and thumbnail through another replica.
|
|
response = c.call(
|
|
"POST",
|
|
"/api/v1/chats/" + room["id"] + "/attachments?filename=cross.png",
|
|
202,
|
|
actor=alice,
|
|
raw=data,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
cross = response["json"]
|
|
accepted.append(cross["id"])
|
|
ready(cross)
|
|
uploader = response["headers"]["X-Instance-Id"]
|
|
process = next(v for v in instances() if v["instance"] == uploader)
|
|
subprocess.run(
|
|
["docker", "stop", "--time", "3", process["id"]], check=True, capture_output=True
|
|
)
|
|
try:
|
|
wait_lb(args.base_url, count=1)
|
|
download = c.call("GET", "/api/v1/attachments/" + cross["id"] + "/content", actor=bob)
|
|
check(
|
|
download["bytes"] == data and download["headers"]["X-Instance-Id"] != uploader,
|
|
"Other replica could not serve the original",
|
|
)
|
|
c.call(
|
|
"GET",
|
|
"/api/v1/attachments/" + cross["id"] + "/content?variant=thumbnail",
|
|
actor=bob,
|
|
)
|
|
cross_proof = {
|
|
"uploader": uploader,
|
|
"stopped": inspect(process["id"]),
|
|
"reader": download["headers"]["X-Instance-Id"],
|
|
}
|
|
finally:
|
|
subprocess.run(["docker", "start", process["id"]], check=True, capture_output=True)
|
|
wait_lb(args.base_url)
|
|
logs = compose("logs", "--no-color", "api", "worker", "relay", "lb")
|
|
check(
|
|
not any(
|
|
actor["token"] in logs or actor["registration"]["password"] in logs
|
|
for actor in (alice, bob, eve)
|
|
),
|
|
"Credential in logs",
|
|
)
|
|
states = query(
|
|
"SELECT id,storage_state FROM attachments WHERE id=ANY(%s::uuid[])", (accepted,)
|
|
)
|
|
check(
|
|
len(states) == len(accepted) and all(s["storage_state"] == "ready" for s in states),
|
|
"Confirmed work lost",
|
|
)
|
|
stats = compose(
|
|
"exec",
|
|
"-T",
|
|
"worker",
|
|
"python",
|
|
"-c",
|
|
"import json;from pathlib import Path;print(json.dumps({"
|
|
"n:Path('/sys/fs/cgroup/'+n).read_text().strip() "
|
|
"for n in ('memory.current','memory.peak','memory.events','cpu.stat')}))",
|
|
)
|
|
result = {
|
|
"anonymous_s3_status": 403,
|
|
"denied_iam_actions": denied,
|
|
"cross_replica": cross_proof,
|
|
"track": 2,
|
|
"delivery": "at-least-once",
|
|
"effect_key": ["attachment_id", "pipeline_version", "variant"],
|
|
"worker_concurrency": config.worker_concurrency,
|
|
"prefetch_per_slot": 1,
|
|
"job_timeout_seconds": config.job_timeout,
|
|
"lease_seconds": config.lease_seconds,
|
|
"max_attempts": config.job_max_attempts,
|
|
"scenarios": records,
|
|
"accepted": len(accepted),
|
|
"lost_confirmed": 0,
|
|
"stale_overwrites": 0,
|
|
"worker_cgroup": json.loads(stats),
|
|
"logs_credentials_absent": True,
|
|
}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
|
|
print("PASS: all acknowledged work ready; no stale overwrites", flush=True)
|
|
finally:
|
|
compose("start", "worker", "relay", "broker", "api")
|
|
broker.close()
|
|
store.close()
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|