Files
mtusi2026lab5/scripts/lab5/track2.py
T

452 lines
22 KiB
Python

#!/usr/bin/env python3
"""Run only on the disposable lab5 fault-overlay Compose stand. Retains volumes."""
import argparse
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import struct
import subprocess
import sys
import time
from urllib.parse import urlsplit
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
MARKERS = "/tmp/lab5-faults/"
VARIANTS = {"thumbnail", "crop", "optimized"}
def digest(data):
return hashlib.sha256(data).hexdigest()
def fixture(seed):
"""A deterministic textured PNG produces different WebP bytes at the two qualities."""
size = 160
rows = bytearray()
for y in range(size):
rows.append(0)
for x in range(size):
rows.extend(((x * 3 + y * 7 + seed) % 256, (x * y + seed) % 256, (x ^ y ^ seed) % 256))
def chunk(kind, data):
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b""))
def command(args, *, allow_failure=False, binary=False, timeout=60):
proc = subprocess.run(args, cwd=ROOT, capture_output=True, timeout=timeout)
if proc.returncode and not allow_failure:
raise Failure(f"{args[:4]} exited {proc.returncode}: " + proc.stderr.decode(errors="replace")[-600:])
if allow_failure:
return proc
return proc.stdout if binary else proc.stdout.decode().strip()
def wait(test, label, timeout=90):
end = time.monotonic() + timeout
while time.monotonic() < end:
value = test()
if value:
return value
time.sleep(0.16)
raise Failure("timed out waiting for " + label)
class Experiment:
def __init__(self, base_url, project):
self.compose = ["docker", "compose", "-p", project, "-f", "compose.yaml", "-f", "compose.faults.yaml"]
address = urlsplit(base_url)
port = self.dc("port", "lb", "8080").splitlines()[0].rsplit(":", 1)[-1]
check(address.hostname in {"127.0.0.1", "localhost"} and str(address.port) == port,
"base URL must match the explicit local Compose stand")
self.worker = self.ids("worker")[0]
self.s3 = self.ids("s3")[0]
self.bucket = self.exe(self.worker, "printenv", "S3_BUCKET")
self.apis = self.ids("api")
check(len(self.apis) >= 2, "two API replicas required")
check("--barriers=" + MARKERS.rstrip("/") in command(["docker", "inspect", "-f", "{{json .Config.Cmd}}", self.worker]),
"worker must run with compose.faults.yaml")
for api in self.apis:
check(self.exe(api, "printenv", "LAB5_BARRIERS") == MARKERS.rstrip("/"), "API barrier overlay required")
self.client = Client(base_url)
self.actor, self.reader, _, self.chat = self.client.bootstrap()
self.content = fixture(41)
self.stopped = set()
self.armed = set()
def dc(self, *args, **kwargs):
return command(self.compose + list(args), **kwargs)
def ids(self, service):
ids = self.dc("ps", "-q", service).splitlines()
check(bool(ids), service + " must be running")
return ids
def exe(self, container, *args, **kwargs):
return command(["docker", "exec", container, *args], **kwargs)
def sql(self, statement):
return self.dc("exec", "-T", "postgres", "psql", "-XqAt", "-v", "ON_ERROR_STOP=1",
"-U", "messenger", "-d", "messenger", "-c", statement)
def marker(self, container, stem, operation):
path = MARKERS + stem
if operation == "arm":
self.exe(container, "mkdir", "-p", MARKERS)
self.armed.add((container, stem))
self.exe(container, "touch", path + ".arm")
elif operation == "hit":
return self.exe(container, "test", "-f", path + ".hit", allow_failure=True).returncode == 0
elif operation == "release":
self.exe(container, "rm", "-f", path + ".arm", path + ".hit", path + ".id")
self.armed.discard((container, stem))
def arm(self, stem, containers):
for container in containers:
self.marker(container, stem, "arm")
def hit(self, stem, containers):
return wait(lambda: next((c for c in containers if self.marker(c, stem, "hit")), None), stem)
def stop(self, service):
self.stopped.add(service)
self.dc("stop", "-t", "3", service)
def start(self, service):
self.dc("start", service)
self.stopped.discard(service)
def crash(self, container):
command(["docker", "kill", "--signal", "KILL", container])
command(["docker", "start", container])
def cli(self, *args, **kwargs):
return self.exe(self.worker, "/usr/local/bin/worker", *args, **kwargs)
def object_hash(self,key):
# Read through the storage process's loopback-only filer endpoint. This
# can observe a worker's PUT before HTTP publication has committed.
url = "http://127.0.0.1:8888/buckets/" + self.bucket + "/" + key
fetched = self.exe(self.s3,"wget","-SO-",url,allow_failure=True)
if fetched.returncode:
check(b"404 Not Found" in fetched.stderr, "cannot inspect filer object " + key)
return None
return digest(fetched.stdout)
def upload(self, data=None):
data = self.content if data is None else data
att = self.client.call("POST", "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=track2.png", 202,
actor=self.actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment")["json"]
check(att["status"] == "queued" and att["sha256"] == digest(data), "invalid accepted upload")
return str(uuid.UUID(att["id"]))
def state(self, aid):
return json.loads(self.sql(f"""
SELECT json_build_object('target',a.target_version,'published',a.published_version,
'jobs',(SELECT json_agg(json_build_object('version',j.pipeline_version,'id',j.job_id,
'attempt',j.attempt,'state',j.state,'error',j.error_code,'confirmed',j.published_at IS NOT NULL)
ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),
'variant_count',(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id))
FROM attachments a WHERE a.id='{aid}'"""))
def result(self, aid, data=None):
data = self.content if data is None else data
att = self.client.wait_ready(self.reader, {"id": aid}, 120)
base = "/api/v1/attachments/" + aid + "/content"
original = self.client.call("GET", base, actor=self.reader)["bytes"]
check(original == data, "accepted original changed")
hashes = {}
for variant in att["variants"]:
name = variant["name"]
obj = self.client.call("GET", base + "?variant=" + name, actor=self.reader)
h = digest(obj["bytes"])
check(h == variant["sha256"] and len(obj["bytes"]) == variant["size_bytes"], "variant bytes differ from metadata")
check(name not in hashes, "duplicate public variant")
hashes[name] = h
check(set(hashes) == VARIANTS, "missing public variant")
state = self.state(aid)
rows = self.sql(f"""SELECT variant||' '||encode(sha256,'hex') FROM attachment_variants
WHERE attachment_id='{aid}' AND pipeline_version={int(state['published'])}""")
check(dict(row.split() for row in rows.splitlines()) == hashes, "SQL and downloaded bytes differ")
return {"attachment": aid, "original_sha256": digest(original), "variant_sha256": hashes, **state}
def queued(self):
self.stop("broker")
aid = self.upload()
state = self.state(aid)
check(state["jobs"][0]["state"] == "queued" and not state["jobs"][0]["confirmed"], "expected pending outbox")
self.client.call("GET", "/api/v1/attachments/" + aid + "/content", 409, actor=self.reader)
return aid
def staged_crash(self):
data = fixture(uuid.uuid4().int % 255)
marker = digest(data) + "-staged"
self.arm(marker, self.apis)
with ThreadPoolExecutor(max_workers=1) as pool:
future = pool.submit(self.upload, data)
owner = self.hit(marker, self.apis)
path = MARKERS + marker + ".id"
def staged_id():
p = self.exe(owner, "cat", path, allow_failure=True)
return p.stdout.decode().strip() if p.returncode == 0 else None
aid = str(uuid.UUID(wait(staged_id, "uncommitted staging UUID")))
self.crash(owner)
try:
future.result()
except Failure:
pass
else:
raise Failure("interrupted staging unexpectedly returned 202")
for api in self.apis:
self.marker(api, marker, "release")
rows = int(self.sql(f"SELECT count(*) FROM attachments WHERE id='{aid}'"))
check(rows == 0, "staged-only upload acquired a logical attachment")
check(self.object_hash("staging/" + aid) == digest(data), "crashed upload did not leave the expected staged bytes")
return {"unaccepted": aid, "source_sha256": digest(data), "attachment_rows": rows}
def outbox_restart(self):
aid = self.queued()
message = self.client.message(self.actor, self.chat, "Принято без брокера", attachments=[aid])["json"]
before = self.state(aid)
self.dc("restart", "api")
def api_ready():
try:
return self.client.call("GET", "/health/ready", schema="Health")["status"] == 200
except Failure:
return False
wait(api_ready, "API readiness without broker")
check(self.state(aid) == before, "durable intent changed during API restart")
self.start("broker")
result = self.result(aid)
check(result["jobs"][0]["id"] == before["jobs"][0]["id"], "restart recreated job")
path = "/api/v1/chats/" + self.chat["id"] + "/messages"
history = self.client.call("GET", path, actor=self.reader, schema="MessagePage")["json"]["items"]
check(message in history, "confirmed message disappeared")
return {"before": before, "message": message["id"], "after": result}
def published_crash(self):
aid = self.queued()
self.stop("worker")
marker = aid + "-v1-published"
self.arm(marker, self.apis)
self.start("broker")
owner = self.hit(marker, self.apis)
for api in self.apis:
if api != owner:
self.marker(api, marker, "release")
before = self.state(aid)
check(not before["jobs"][0]["confirmed"], "SQL confirmation already committed")
self.crash(owner)
self.marker(owner, marker, "release")
wait(lambda: self.state(aid)["jobs"][0]["confirmed"], "republished envelope")
def queued_copies():
rows = json.loads(self.dc("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues",
"name", "messages_ready", "--formatter=json"))
copies = next((row["messages_ready"] for row in rows if row["name"] == "attachment.jobs"), 0)
return copies if copies >= 2 else 0
copies = wait(queued_copies, "two confirmed broker copies while consumers are stopped")
confirmed = self.state(aid)
self.dc("restart", "broker")
def after_restart():
try:
return queued_copies()
except Failure:
return 0
restored = wait(after_restart, "durable queue after broker restart", 120)
check(copies == restored and self.state(aid) == confirmed, "confirmed queue or SQL job changed across broker restart")
self.start("worker")
result = self.result(aid)
check(result["variant_count"] == 3 and result["jobs"][0]["attempt"] == 1, "publication crash duplicated the effect")
return {"broker_confirm_before_sql_commit": before, "confirmed_job_before_broker_restart": confirmed,
"broker_messages_before_restart":copies,"broker_messages_after_restart":restored,"after": result}
def worker_crash(self, phase, reference):
aid = self.queued()
marker = aid + "-v1-" + phase
self.arm(marker, [self.worker])
self.start("broker")
self.hit(marker, [self.worker])
before = self.state(aid)
check(before["variant_count"] == (3 if phase == "committed" else 0), "crash occurred on the wrong side of SQL commit")
observed = {name: self.object_hash(f"variants/{aid}/1/{name}/{value}") for name,value in reference.items()}
expected = {name:None for name in VARIANTS} if phase == "claimed" else reference
check(observed == expected, "crash occurred on the wrong side of storage writes")
if phase == "committed":
duplicate = aid + "-v1-duplicate"
self.arm(duplicate, [self.worker])
self.crash(self.worker)
self.marker(self.worker, marker, "release")
if phase == "committed":
self.hit(duplicate, [self.worker])
self.marker(self.worker, duplicate, "release")
result = self.result(aid)
want = 1 if phase == "committed" else 2
check(result["jobs"][0]["attempt"] == want and result["variant_count"] == 3, "incorrect crash recovery")
return {"phase": phase, "before": before, "storage_sha256_at_barrier": observed, "after": result}
def duplicate_ack(self, aid, version):
prefix = f"{aid}-v{version}-"
for phase in ["duplicate", "acked"]:
self.arm(prefix + phase, [self.worker])
self.cli("duplicate", aid, str(version), "1")
for phase in ["duplicate", "acked"]:
self.hit(prefix + phase, [self.worker])
self.marker(self.worker, prefix + phase, "release")
def lease_takeover(self):
aid = self.queued()
objects = aid + "-v1-objects"
committed = aid + "-v1-committed"
acked = aid + "-v1-acked"
self.arm(objects, [self.worker])
self.arm(committed, [self.worker])
self.start("broker")
self.hit(objects, [self.worker])
first = self.state(aid)
self.hit(committed, [self.worker])
winner = self.result(aid)
check(first["jobs"][0]["attempt"] == 1 and winner["jobs"][0]["attempt"] == 2
and first["jobs"][0]["id"] == winner["jobs"][0]["id"], "lease takeover changed logical work")
for barrier in [committed, objects]:
self.arm(acked, [self.worker])
self.marker(self.worker, barrier, "release")
self.hit(acked, [self.worker])
self.marker(self.worker, acked, "release")
after = self.result(aid)
check(after == winner and after["variant_count"] == 3, "expired worker modified published result")
return {"first_held": first, "second_committed": winner, "after_first_ack": after}
def reordered(self, v1_hashes):
aid = self.queued()
marker = aid + "-v1-objects"
self.arm(marker, [self.worker])
self.start("broker")
self.hit(marker, [self.worker])
self.duplicate_ack(aid, 1)
self.cli("reprocess", aid, "2")
before = self.result(aid)
check(before["published"] == 2 and before["variant_count"] == 3 and before["jobs"][0]["state"] == "superseded"
and before["jobs"][0]["attempt"] == 1,
"new version did not finish while first worker held its result")
check(v1_hashes["optimized"] != before["variant_sha256"]["optimized"], "fixture must distinguish the profiles")
acked = aid + "-v1-acked"
self.arm(acked, [self.worker])
self.marker(self.worker, marker, "release")
self.hit(acked, [self.worker])
self.marker(self.worker, acked, "release")
for version in [2, 1]:
self.duplicate_ack(aid, version)
after = self.result(aid)
check(after == before, "stale commit or delivery changed new result")
return {"reference_v1_webp": v1_hashes["optimized"], "before_releasing_v1": before,
"after_acks_in_order": [2, 1], "after": after}
def retry_replay(self):
aid = self.queued()
real = "staging/" + aid
self.sql(f"UPDATE attachment_jobs SET source_key='staging/{uuid.uuid4()}' WHERE attachment_id='{aid}'")
self.start("broker")
wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "retry exhaustion", 180)
failed = self.state(aid)
limit = int(self.exe(self.worker, "printenv", "JOB_MAX_ATTEMPTS"))
job = failed["jobs"][0]
check(job["attempt"] == limit and job["error"] == "source_missing", "wrong failure budget")
entries = json.loads(self.cli("failed"))
check(any(row["job_id"] == job["id"] for row in entries), "missing failed job")
self.sql(f"UPDATE attachment_jobs SET source_key='{real}' WHERE attachment_id='{aid}'")
self.cli("replay", aid, "track2")
result = self.result(aid)
check(result["jobs"][0]["attempt"] == limit + 1, "replay reused a stale attempt")
rejected = self.cli("replay", aid, "track2", allow_failure=True)
check(rejected.returncode != 0, "accepted replay of successful job")
count = int(self.sql(f"SELECT count(*) FROM attachment_replays WHERE job_id='{job['id']}' AND actor='track2'"))
check(count == 1, "incorrect audit count")
return {"injection": "absent source key assigned only to synthetic job", "exhausted": failed, "after": result, "audit_rows": count}
def permanent_failure(self, other_sha):
self.stop("broker")
aid = self.upload(fixture(19))
self.sql(f"UPDATE attachment_jobs SET source_key='blobs/{other_sha}' WHERE attachment_id='{aid}'")
self.start("broker")
wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "permanent source mismatch")
failed = self.state(aid)
j = failed["jobs"][0]
check(j["attempt"] == 1 and j["error"] == "source_mismatch" and failed["variant_count"] == 0,
"permanent failure consumed additional attempts")
self.sql(f"UPDATE attachment_jobs SET source_key='staging/{aid}' WHERE attachment_id='{aid}'")
return {"injection": "different valid original assigned only to synthetic job", "failed": failed}
def cleanup_stage(self, aid, created):
# Run normal GC after its real minimum grace. No object timestamps are forged.
remaining = created + 125 - time.monotonic()
if remaining > 0:
time.sleep(remaining)
def run_gc(apply):
args = ["run", "--rm", "--no-deps", "-T", "reconcile", "--grace=2m"]
if apply:
args.append("--apply")
return [json.loads(line) for line in self.dc(*args, timeout=180).splitlines()]
def matches(rows):
return [row for row in rows if row.get("key") == "staging/" + aid]
before = matches(run_gc(False))
check(len(before) == 1 and before[0]["kind"] == "stale_staging", "unaccepted stage not reported")
applied = matches(run_gc(True))
check(len(applied) == 1 and applied[0]["applied"], "unaccepted stage not removed")
check(not matches(run_gc(False)), "removed stage still listed")
return {"before": before[0], "applied": applied[0], "after_findings": 0}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", required=True)
parser.add_argument("--project", choices=["lab5"], required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
exp = Experiment(args.base_url, args.project)
started = time.monotonic()
report = {"project": args.project, "complete": False, "scenarios": {}, "seconds": {},
"started_at":datetime.now(timezone.utc).isoformat()}
args.output.parent.mkdir(parents=True, exist_ok=True)
def save():
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
def record(name, callback):
print(name, flush=True)
beginning = time.monotonic()
report["scenarios"][name] = callback()
report["seconds"][name] = round(time.monotonic() - beginning, 3)
save()
return report["scenarios"][name]
save()
try:
stage = record("staging_before_commit", exp.staged_crash)
created = time.monotonic()
accepted = record("commit_before_publish", exp.outbox_restart)["after"]
record("broker_confirm_before_sql", exp.published_crash)
for phase in ["claimed", "objects", "committed"]:
record("worker_crash_" + phase, lambda: exp.worker_crash(phase, accepted["variant_sha256"]))
record("same_version_lease_takeover", exp.lease_takeover)
record("parallel_duplicate_and_reorder", lambda: exp.reordered(accepted["variant_sha256"]))
record("retry_exhaustion_replay", exp.retry_replay)
record("permanent_failure", lambda: exp.permanent_failure(accepted["original_sha256"]))
record("unaccepted_staging_cleanup", lambda: exp.cleanup_stage(stage["unaccepted"], created))
report["complete"] = True
report["finished_at"] = datetime.now(timezone.utc).isoformat()
report["elapsed_seconds"] = round(time.monotonic() - started, 3)
save()
finally:
for service in sorted(exp.stopped):
exp.start(service)
for container, stem in list(exp.armed):
exp.marker(container, stem, "release")
if __name__ == "__main__":
main()