forked from mtusi2026/mtusi2026lab5
321 lines
14 KiB
Python
321 lines
14 KiB
Python
"""Fault experiment on a dedicated lab5 Compose stand with synthetic data."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check
|
|
|
|
|
|
def run(*args, expected=0, timeout=120):
|
|
proc = subprocess.run(
|
|
args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace", timeout=timeout,
|
|
)
|
|
if proc.returncode != expected:
|
|
raise Failure(f"{args[:5]} exit={proc.returncode}, expected={expected}")
|
|
return proc.stdout.strip()
|
|
|
|
|
|
def compose(*args, **kwargs):
|
|
return run("docker", "compose", "--ansi", "never", *args, **kwargs)
|
|
|
|
|
|
def sql(query):
|
|
return compose("exec", "-T", "db", "psql", "-XAt", "-v", "ON_ERROR_STOP=1",
|
|
"-U", "messenger", "-d", "messenger", "-c", query)
|
|
|
|
|
|
def broker(*args):
|
|
output = compose(
|
|
"exec", "-T", "broker", "/bin/sh", "-ec",
|
|
'export REDISCLI_AUTH="$(cat /run/secrets/broker_password)"; exec valkey-cli --json "$@"',
|
|
"sh", *args,
|
|
)
|
|
return json.loads(output)
|
|
|
|
|
|
def until(fn, timeout=90):
|
|
start = time.monotonic()
|
|
while time.monotonic() - start < timeout:
|
|
try:
|
|
result = fn()
|
|
except Failure:
|
|
result = None
|
|
if result:
|
|
return result
|
|
time.sleep(0.3)
|
|
raise Failure(f"condition timed out after {timeout}s")
|
|
|
|
|
|
def job(att, version=1):
|
|
return json.loads(sql(f"""
|
|
SELECT json_build_object('id',id,'state',state,'attempt',attempt,'error',error_code,
|
|
'published',published,'created',to_char(created_at AT TIME ZONE 'UTC','YYYY-MM-DD"T"HH24:MI:SS.US"Z"'),
|
|
'updated',updated_at,'delay_ms',round(extract(epoch FROM (next_at-updated_at))*1000))
|
|
FROM image_jobs WHERE attachment_id='{att['id']}' AND version={version}
|
|
"""))
|
|
|
|
|
|
def carrier(att, version):
|
|
row = job(att, version)
|
|
return {
|
|
"schema_version": 1, "job_id": row["id"], "type": "attachment.process.v1",
|
|
"attachment_id": att["id"], "pipeline_version": version,
|
|
"attempt": row["attempt"], "created_at": row["created"],
|
|
}
|
|
|
|
|
|
def entries(att):
|
|
return [json.loads(fields[1]) for _, fields in broker("XRANGE", "attachment-jobs", "-", "+")
|
|
if json.loads(fields[1])["attachment_id"] == att["id"]]
|
|
|
|
|
|
def publish(payload):
|
|
return broker("XADD", "attachment-jobs", "*", "job", json.dumps(payload, separators=(",", ":")))
|
|
|
|
|
|
def s3stat(key):
|
|
setup = ('mc alias set local http://s3:9000 "$(cat /run/secrets/s3_root_user)" '
|
|
'"$(cat /run/secrets/s3_root_password)" >/dev/null; '
|
|
'mc stat --json "local/messenger-attachments/$1"')
|
|
# mc reports an absent object in JSON on stdout.
|
|
args = ("docker", "compose", "--ansi", "never", "run", "--rm", "--no-deps",
|
|
"--entrypoint", "/bin/sh", "s3-init", "-ec", setup, "sh", key)
|
|
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True,
|
|
encoding="utf-8", errors="replace", timeout=60)
|
|
result = json.loads(proc.stdout.strip().splitlines()[-1])
|
|
if proc.returncode:
|
|
check("does not exist" in json.dumps(result).lower(), "unexpected object-store error")
|
|
return None
|
|
return result
|
|
|
|
|
|
def fault(service, phase, att):
|
|
out = compose("run", "--rm", "--no-deps", "-e", f"LAB5_FAULT={phase}",
|
|
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", service, expected=86)
|
|
check('"test crash"' in out and f'"phase":"{phase}"' in out, "fault marker missing")
|
|
|
|
|
|
def generations(*services):
|
|
ids = compose("ps", "-q", *services).splitlines()
|
|
return dict(line.split("|", 1) for line in run(
|
|
"docker", "inspect", "--format", "{{.Id}}|{{.State.StartedAt}}", *ids,
|
|
).splitlines())
|
|
|
|
|
|
class Experiment:
|
|
def __init__(self):
|
|
self.client = Client()
|
|
self.alice, self.bob, self.eve, self.chat = self.client.bootstrap()
|
|
self.results = []
|
|
|
|
def record(self, scenario, att, started, **data):
|
|
row = {"scenario": scenario, "attachment_id": att["id"],
|
|
"seconds": round(time.monotonic() - started, 3), **data}
|
|
self.results.append(row)
|
|
print(json.dumps(row, ensure_ascii=False), flush=True)
|
|
|
|
def metadata(self, att):
|
|
return self.client.call("GET", "/api/v1/attachments/" + att["id"],
|
|
actor=self.bob, schema="Attachment")["json"]
|
|
|
|
def content(self, att, variant, status=200, actor=None):
|
|
return self.client.call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}",
|
|
status, actor=actor or self.bob)
|
|
|
|
def ready(self, att, raw):
|
|
result = self.client.wait_ready(self.bob, att, wait_seconds=155)
|
|
check(self.content(att, "original")["bytes"] == raw, "original changed")
|
|
variants = {v["name"]: v for v in result["variants"]}
|
|
check(len(result["variants"]) == 3 and set(variants) == {"thumbnail", "crop", "optimized"},
|
|
"expected exactly one of each variant")
|
|
for name, (width, height, mime) in {
|
|
"thumbnail": (256, 160, "image/jpeg"), "crop": (128, 128, "image/jpeg"),
|
|
"optimized": (320, 200, "image/webp"),
|
|
}.items():
|
|
variant = variants[name]
|
|
download = self.content(att, name)
|
|
body = download["bytes"]
|
|
check((variant["width"], variant["height"], download["headers"].get_content_type())
|
|
== (width, height, mime), "wrong dimensions or MIME")
|
|
check(variant["size_bytes"] == len(body) and variant["sha256"] == hashlib.sha256(body).hexdigest(),
|
|
"wrong variant size or checksum")
|
|
check(body.startswith(b"\xff\xd8") if mime == "image/jpeg" else
|
|
body[:4] == b"RIFF" and body[8:12] == b"WEBP", "wrong encoding")
|
|
return result
|
|
|
|
def restart(self):
|
|
compose("stop", "worker")
|
|
att, raw = self.client.attachment(self.alice, self.chat)
|
|
started = time.monotonic()
|
|
check(self.metadata(att)["status"] == "queued", "job ran while worker stopped")
|
|
self.content(att, "original", 409)
|
|
self.content(att, "original", 404, self.eve)
|
|
message = self.client.message(self.alice, self.chat, "", attachments=[att["id"]])["json"]
|
|
before = until(lambda: entries(att))
|
|
services = ("api", "broker")
|
|
old = generations(*services)
|
|
check(len(old) >= 3, "expected two API replicas and broker")
|
|
compose("restart", *services)
|
|
new = generations(*services)
|
|
check(old.keys() == new.keys() and all(old[key] != new[key] for key in old), "restart not observed")
|
|
check(until(lambda: entries(att)) == before, "durable broker entry changed")
|
|
until(lambda: self.client.call("GET", "/health/ready", schema="Health"))
|
|
compose("start", "worker")
|
|
self.ready(att, raw)
|
|
until(lambda: not entries(att))
|
|
self.record("api_broker_restart_with_worker_stopped", att, started,
|
|
message_id=message["id"], job_id=before[0]["job_id"], processes_restarted=len(old))
|
|
|
|
def publish_crash(self, phase):
|
|
compose("stop", "dispatcher", "worker")
|
|
att, raw = self.client.attachment(self.alice, self.chat)
|
|
started = time.monotonic()
|
|
fault("dispatcher", phase, att)
|
|
old = job(att)
|
|
check(old["state"] == "queued" and not old["published"], "outbox lost pending intent")
|
|
observed = entries(att)
|
|
check(len(observed) == (phase == "after_publish"), "incorrect broker state at crash")
|
|
# Keep the worker stopped until both XADDs are visible in the ambiguous case.
|
|
compose("start", "dispatcher")
|
|
count = 2 if phase == "after_publish" else 1
|
|
until(lambda: len(entries(att)) == count)
|
|
check(all(entry == carrier(att, 1) for entry in entries(att)), "unexpected carrier")
|
|
compose("start", "worker")
|
|
self.ready(att, raw)
|
|
until(lambda: not entries(att))
|
|
check(job(att)["attempt"] == 1, "duplicate delivery consumed an attempt")
|
|
self.record(phase, att, started, copies=count, attempt=1)
|
|
|
|
def worker_crash(self, phase):
|
|
compose("stop", "worker")
|
|
att, raw = self.client.attachment(self.alice, self.chat)
|
|
started = time.monotonic()
|
|
fault("worker", phase, att)
|
|
old = job(att)
|
|
path = f"original/{att['id']}/v1"
|
|
written = s3stat(path)
|
|
check((written is not None) == (phase != "before_effect"), "incorrect S3 state at crash")
|
|
check(old["state"] == ("ready" if phase == "after_commit" else "processing"), "incorrect job state at crash")
|
|
check(broker("XPENDING", "attachment-jobs", "image-workers")[0] > 0, "missing unacked delivery")
|
|
compose("start", "worker")
|
|
self.ready(att, raw)
|
|
until(lambda: not entries(att), timeout=155)
|
|
if written:
|
|
final = s3stat(path)
|
|
check(final["lastModified"] == written["lastModified"] and final["etag"] == written["etag"],
|
|
"retry rewrote an existing effect")
|
|
check(job(att)["attempt"] == (1 if phase == "after_commit" else 2), "wrong attempt after recovery")
|
|
until(lambda: s3stat(f"staging/{att['id']}") is None, timeout=50)
|
|
self.record(phase, att, started, state_at_crash=old["state"], object_at_crash=written is not None,
|
|
final_attempt=job(att)["attempt"], staging_removed=True)
|
|
|
|
def replay(self, phase):
|
|
compose("stop", "worker")
|
|
att, raw = self.client.attachment(self.alice, self.chat)
|
|
started = time.monotonic()
|
|
name = f"lab5-fault-{uuid.uuid4().hex[:12]}"
|
|
compose("run", "-d", "--no-deps", "--name", name, "-e", f"LAB5_FAULT={phase}",
|
|
"-e", f"LAB5_FAULT_ATTACHMENT={att['id']}", "worker")
|
|
delays = {}
|
|
try:
|
|
def failure():
|
|
row = job(att)
|
|
if row["attempt"] > 1 and row["delay_ms"] > 0:
|
|
delays.setdefault(row["attempt"], row["delay_ms"])
|
|
return row if row["state"] == "failed" else None
|
|
|
|
failed = until(failure, timeout=155)
|
|
finally:
|
|
run("docker", "stop", name)
|
|
run("docker", "rm", name)
|
|
code = "invalid_image" if phase == "invalid_image" else "retry_exhausted"
|
|
meta = self.metadata(att)
|
|
check(meta["status"] == "failed" and meta["error_code"] == code, "incorrect final failure")
|
|
if phase == "invalid_image":
|
|
check(failed["attempt"] == 1, "permanent error was retried")
|
|
else:
|
|
config = json.loads(compose("config", "--format", "json"))
|
|
max_attempts = int(config["services"]["worker"]["environment"]["JOB_MAX_ATTEMPTS"])
|
|
check(failed["attempt"] == max_attempts, "incorrect retry limit")
|
|
check(set(delays) == set(range(2, max_attempts + 1)), "missed retry schedule")
|
|
for attempt, delay in delays.items():
|
|
base = 1000 << min(attempt - 2, 5)
|
|
check(base - 5 <= delay < 2 * base, "retry delay outside jitter window")
|
|
self.content(att, "original", 409)
|
|
self.client.call("POST", f"/api/v1/chats/{self.chat['id']}/messages", 409, actor=self.alice,
|
|
headers={"Idempotency-Key": uuid.uuid4().hex}, body={"text": "", "attachment_ids": [att["id"]]})
|
|
def operator(*args, **kwargs):
|
|
return compose("run", "--rm", "--no-deps", "dispatcher", "jobs", *args, **kwargs)
|
|
check(failed["id"] in operator("failed"), "failed registry did not contain job")
|
|
operator("replay", failed["id"])
|
|
check(self.metadata(att)["status"] == "queued", "replay failed to queue job")
|
|
compose("start", "worker")
|
|
self.ready(att, raw)
|
|
operator("replay", failed["id"], expected=1)
|
|
audit = sql(f"SELECT count(*) FROM image_job_replays WHERE job_id='{failed['id']}'")
|
|
check(audit == "1" and job(att)["attempt"] == failed["attempt"] + 1, "replay audit/attempt mismatch")
|
|
self.record(phase + "_replay", att, started, failed_attempt=failed["attempt"],
|
|
error_code=code, retry_delays_ms=delays, replay_audit=int(audit))
|
|
|
|
def version_order(self):
|
|
att, raw = self.client.attachment(self.alice, self.chat)
|
|
started = time.monotonic()
|
|
v1 = self.ready(att, raw)
|
|
old_carrier = carrier(att, 1)
|
|
old_object = s3stat(f"optimized/{att['id']}/v1")
|
|
until(lambda: not entries(att))
|
|
compose("stop", "worker")
|
|
compose("run", "--rm", "--no-deps", "dispatcher", "jobs", "upgrade", att["id"])
|
|
until(lambda: job(att, 2)["published"])
|
|
new_carrier = carrier(att, 2)
|
|
publish(old_carrier)
|
|
publish(new_carrier)
|
|
check([entry["pipeline_version"] for entry in entries(att)] == [2, 1, 2], "reorder not injected")
|
|
check(self.metadata(att) == v1, "upgrade hid the previous result")
|
|
compose("start", "worker")
|
|
until(lambda: sql(f"SELECT result_version FROM attachments WHERE id='{att['id']}'") == "2")
|
|
v2 = self.ready(att, raw)
|
|
until(lambda: not entries(att))
|
|
check(v1["variants"] != v2["variants"], "v2 was not distinct")
|
|
new_object = s3stat(f"optimized/{att['id']}/v2")
|
|
publish(old_carrier)
|
|
publish(new_carrier)
|
|
until(lambda: not entries(att))
|
|
check(self.ready(att, raw) == v2 and job(att, 2)["attempt"] == 1, "stale job changed v2")
|
|
for version, obj in ((1, old_object), (2, new_object)):
|
|
current = s3stat(f"optimized/{att['id']}/v{version}")
|
|
check(current["lastModified"] == obj["lastModified"] and current["etag"] == obj["etag"],
|
|
"duplicate or reordered job rewrote an object")
|
|
self.record("reorder_and_stale_version", att, started, delivery_versions=[2, 1, 2, 1, 2],
|
|
result_version=2, attempts=[job(att)["attempt"], job(att, 2)["attempt"]])
|
|
|
|
|
|
def main():
|
|
experiment = Experiment()
|
|
try:
|
|
experiment.restart()
|
|
for phase in ("before_publish", "after_publish"):
|
|
experiment.publish_crash(phase)
|
|
for phase in ("before_effect", "after_effect", "after_commit"):
|
|
experiment.worker_crash(phase)
|
|
for phase in ("invalid_image", "temporary"):
|
|
experiment.replay(phase)
|
|
experiment.version_order()
|
|
finally:
|
|
compose("start", "dispatcher", "worker")
|
|
(ROOT / "evidence" / "pipeline.json").write_text(
|
|
json.dumps(experiment.results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8",
|
|
)
|
|
print("PASS pipeline fault experiment")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|