459 lines
22 KiB
Python
459 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Run only on the disposable lab5 fault-overlay Compose stand. Retains volumes."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
import uuid
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client, Failure, check # noqa: E402
|
|
|
|
MARKERS = "/tmp/lab5-faults/"
|
|
VARIANTS = {"thumbnail", "crop", "optimized"}
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def fixture(seed):
|
|
"""A deterministic textured PNG produces different WebP bytes at the two qualities."""
|
|
size = 160
|
|
rows = bytearray()
|
|
for y in range(size):
|
|
rows.append(0)
|
|
for x in range(size):
|
|
rows.extend(((x * 3 + y * 7 + seed) % 256, (x * y + seed) % 256, (x ^ y ^ seed) % 256))
|
|
def chunk(kind, data):
|
|
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
|
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0))
|
|
+ chunk(b"IDAT", zlib.compress(rows)) + chunk(b"IEND", b""))
|
|
|
|
|
|
def command(args, *, allow_failure=False, binary=False, timeout=60):
|
|
proc = subprocess.run(args, cwd=ROOT, capture_output=True, timeout=timeout)
|
|
if proc.returncode and not allow_failure:
|
|
raise Failure(f"{args[:4]} exited {proc.returncode}: " + proc.stderr.decode(errors="replace")[-600:])
|
|
if allow_failure:
|
|
return proc
|
|
return proc.stdout if binary else proc.stdout.decode().strip()
|
|
|
|
|
|
def wait(test, label, timeout=90):
|
|
end = time.monotonic() + timeout
|
|
while time.monotonic() < end:
|
|
value = test()
|
|
if value:
|
|
return value
|
|
time.sleep(0.16)
|
|
raise Failure("timed out waiting for " + label)
|
|
|
|
|
|
class Experiment:
|
|
def __init__(self, base_url, project):
|
|
self.compose = ["docker", "compose", "-p", project, "-f", "compose.yaml", "-f", "compose.faults.yaml"]
|
|
address = urlsplit(base_url)
|
|
port = self.dc("port", "lb", "8080").splitlines()[0].rsplit(":", 1)[-1]
|
|
check(address.hostname in {"127.0.0.1", "localhost"} and str(address.port) == port,
|
|
"base URL must match the explicit local Compose stand")
|
|
self.worker = self.ids("worker")[0]
|
|
self.s3 = self.ids("s3")[0]
|
|
self.bucket = self.exe(self.worker, "printenv", "S3_BUCKET")
|
|
self.apis = self.ids("api")
|
|
check(len(self.apis) >= 2, "two API replicas required")
|
|
check("--barriers=" + MARKERS.rstrip("/") in command(["docker", "inspect", "-f", "{{json .Config.Cmd}}", self.worker]),
|
|
"worker must run with compose.faults.yaml")
|
|
for api in self.apis:
|
|
check(self.exe(api, "printenv", "LAB5_BARRIERS") == MARKERS.rstrip("/"), "API barrier overlay required")
|
|
self.client = Client(base_url)
|
|
self.actor, self.reader, _, self.chat = self.client.bootstrap()
|
|
self.content = fixture(41)
|
|
self.stopped = set()
|
|
self.armed = set()
|
|
|
|
def dc(self, *args, **kwargs):
|
|
return command(self.compose + list(args), **kwargs)
|
|
|
|
def ids(self, service):
|
|
ids = self.dc("ps", "-q", service).splitlines()
|
|
check(bool(ids), service + " must be running")
|
|
return ids
|
|
|
|
def exe(self, container, *args, **kwargs):
|
|
return command(["docker", "exec", container, *args], **kwargs)
|
|
|
|
def sql(self, statement):
|
|
return self.dc("exec", "-T", "postgres", "psql", "-XqAt", "-v", "ON_ERROR_STOP=1",
|
|
"-U", "messenger", "-d", "messenger", "-c", statement)
|
|
|
|
def marker(self, container, stem, operation):
|
|
path = MARKERS + stem
|
|
if operation == "arm":
|
|
self.exe(container, "mkdir", "-p", MARKERS)
|
|
self.armed.add((container, stem))
|
|
self.exe(container, "touch", path + ".arm")
|
|
elif operation == "hit":
|
|
return self.exe(container, "test", "-f", path + ".hit", allow_failure=True).returncode == 0
|
|
elif operation == "release":
|
|
self.exe(container, "rm", "-f", path + ".arm", path + ".hit", path + ".id")
|
|
self.armed.discard((container, stem))
|
|
|
|
def arm(self, stem, containers):
|
|
for container in containers:
|
|
self.marker(container, stem, "arm")
|
|
|
|
def hit(self, stem, containers):
|
|
return wait(lambda: next((c for c in containers if self.marker(c, stem, "hit")), None), stem)
|
|
|
|
def stop(self, service):
|
|
self.stopped.add(service)
|
|
self.dc("stop", "-t", "3", service)
|
|
|
|
def start(self, service):
|
|
self.dc("start", service)
|
|
self.stopped.discard(service)
|
|
|
|
def crash(self, container):
|
|
command(["docker", "kill", "--signal", "KILL", container])
|
|
command(["docker", "start", container])
|
|
|
|
def cli(self, *args, **kwargs):
|
|
return self.exe(self.worker, "/usr/local/bin/worker", *args, **kwargs)
|
|
|
|
def object_hash(self,key):
|
|
# Read through the storage process's loopback-only filer endpoint. This
|
|
# can observe a worker's PUT before HTTP publication has committed.
|
|
url = "http://127.0.0.1:8888/buckets/" + self.bucket + "/" + key
|
|
fetched = self.exe(self.s3,"wget","-SO-",url,allow_failure=True)
|
|
if fetched.returncode:
|
|
check(b"404 Not Found" in fetched.stderr, "cannot inspect filer object " + key)
|
|
return None
|
|
return digest(fetched.stdout)
|
|
|
|
def upload(self, data=None):
|
|
data = self.content if data is None else data
|
|
att = self.client.call("POST", "/api/v1/chats/" + self.chat["id"] + "/attachments?filename=track2.png", 202,
|
|
actor=self.actor, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment")["json"]
|
|
check(att["status"] == "queued" and att["sha256"] == digest(data), "invalid accepted upload")
|
|
return str(uuid.UUID(att["id"]))
|
|
|
|
def state(self, aid):
|
|
return json.loads(self.sql(f"""
|
|
SELECT json_build_object('target',a.target_version,'published',a.published_version,
|
|
'jobs',(SELECT json_agg(json_build_object('version',j.pipeline_version,'id',j.job_id,
|
|
'attempt',j.attempt,'state',j.state,'error',j.error_code,'confirmed',j.published_at IS NOT NULL)
|
|
ORDER BY j.pipeline_version) FROM attachment_jobs j WHERE j.attachment_id=a.id),
|
|
'variant_count',(SELECT count(*) FROM attachment_variants WHERE attachment_id=a.id))
|
|
FROM attachments a WHERE a.id='{aid}'"""))
|
|
|
|
def result(self, aid, data=None):
|
|
data = self.content if data is None else data
|
|
att = self.client.wait_ready(self.reader, {"id": aid}, 120)
|
|
base = "/api/v1/attachments/" + aid + "/content"
|
|
original = self.client.call("GET", base, actor=self.reader)["bytes"]
|
|
check(original == data, "accepted original changed")
|
|
hashes = {}
|
|
for variant in att["variants"]:
|
|
name = variant["name"]
|
|
obj = self.client.call("GET", base + "?variant=" + name, actor=self.reader)
|
|
h = digest(obj["bytes"])
|
|
check(h == variant["sha256"] and len(obj["bytes"]) == variant["size_bytes"], "variant bytes differ from metadata")
|
|
check(name not in hashes, "duplicate public variant")
|
|
hashes[name] = h
|
|
check(set(hashes) == VARIANTS, "missing public variant")
|
|
state = self.state(aid)
|
|
rows = self.sql(f"""SELECT variant||' '||encode(sha256,'hex') FROM attachment_variants
|
|
WHERE attachment_id='{aid}' AND pipeline_version={int(state['published'])}""")
|
|
check(dict(row.split() for row in rows.splitlines()) == hashes, "SQL and downloaded bytes differ")
|
|
return {"attachment": aid, "original_sha256": digest(original), "variant_sha256": hashes, **state}
|
|
|
|
def queued(self):
|
|
self.stop("broker")
|
|
aid = self.upload()
|
|
state = self.state(aid)
|
|
check(state["jobs"][0]["state"] == "queued" and not state["jobs"][0]["confirmed"], "expected pending outbox")
|
|
self.client.call("GET", "/api/v1/attachments/" + aid + "/content", 409, actor=self.reader)
|
|
return aid
|
|
|
|
def staged_crash(self):
|
|
data = fixture(uuid.uuid4().int % 255)
|
|
marker = digest(data) + "-staged"
|
|
self.arm(marker, self.apis)
|
|
with ThreadPoolExecutor(max_workers=1) as pool:
|
|
future = pool.submit(self.upload, data)
|
|
owner = self.hit(marker, self.apis)
|
|
path = MARKERS + marker + ".id"
|
|
def staged_id():
|
|
p = self.exe(owner, "cat", path, allow_failure=True)
|
|
return p.stdout.decode().strip() if p.returncode == 0 else None
|
|
aid = str(uuid.UUID(wait(staged_id, "uncommitted staging UUID")))
|
|
self.crash(owner)
|
|
try:
|
|
future.result()
|
|
except Failure:
|
|
pass
|
|
else:
|
|
raise Failure("interrupted staging unexpectedly returned 202")
|
|
for api in self.apis:
|
|
self.marker(api, marker, "release")
|
|
rows = int(self.sql(f"SELECT count(*) FROM attachments WHERE id='{aid}'"))
|
|
check(rows == 0, "staged-only upload acquired a logical attachment")
|
|
check(self.object_hash("staging/" + aid) == digest(data), "crashed upload did not leave the expected staged bytes")
|
|
return {"unaccepted": aid, "source_sha256": digest(data), "attachment_rows": rows}
|
|
|
|
def outbox_restart(self):
|
|
aid = self.queued()
|
|
message = self.client.message(self.actor, self.chat, "Принято без брокера", attachments=[aid])["json"]
|
|
before = self.state(aid)
|
|
self.dc("restart", "api")
|
|
def api_ready():
|
|
try:
|
|
return self.client.call("GET", "/health/ready", schema="Health")["status"] == 200
|
|
except Failure:
|
|
return False
|
|
wait(api_ready, "API readiness without broker")
|
|
check(self.state(aid) == before, "durable intent changed during API restart")
|
|
self.start("broker")
|
|
result = self.result(aid)
|
|
check(result["jobs"][0]["id"] == before["jobs"][0]["id"], "restart recreated job")
|
|
path = "/api/v1/chats/" + self.chat["id"] + "/messages"
|
|
history = self.client.call("GET", path, actor=self.reader, schema="MessagePage")["json"]["items"]
|
|
check(message in history, "confirmed message disappeared")
|
|
return {"before": before, "message": message["id"], "after": result}
|
|
|
|
def published_crash(self):
|
|
aid = self.queued()
|
|
self.stop("worker")
|
|
marker = aid + "-v1-published"
|
|
self.arm(marker, self.apis)
|
|
self.start("broker")
|
|
owner = self.hit(marker, self.apis)
|
|
for api in self.apis:
|
|
if api != owner:
|
|
self.marker(api, marker, "release")
|
|
before = self.state(aid)
|
|
check(not before["jobs"][0]["confirmed"], "SQL confirmation already committed")
|
|
# Attaching a queued file must finish even while the publisher holds its row lock.
|
|
linked = self.client.message(self.actor, self.chat, "", attachments=[aid])
|
|
check(linked["json"]["attachment_ids"] == [aid], "message lost its attachment")
|
|
self.crash(owner)
|
|
self.marker(owner, marker, "release")
|
|
wait(lambda: self.state(aid)["jobs"][0]["confirmed"], "republished envelope")
|
|
def queued_copies():
|
|
rows = json.loads(self.dc("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues",
|
|
"name", "messages_ready", "--formatter=json"))
|
|
copies = next((row["messages_ready"] for row in rows if row["name"] == "attachment.jobs"), 0)
|
|
return copies if copies >= 2 else 0
|
|
copies = wait(queued_copies, "two confirmed broker copies while consumers are stopped")
|
|
confirmed = self.state(aid)
|
|
self.dc("restart", "broker")
|
|
def after_restart():
|
|
try:
|
|
return queued_copies()
|
|
except Failure:
|
|
return 0
|
|
restored = wait(after_restart, "durable queue after broker restart", 120)
|
|
check(copies == restored and self.state(aid) == confirmed, "confirmed queue or SQL job changed across broker restart")
|
|
self.start("worker")
|
|
result = self.result(aid)
|
|
check(result["variant_count"] == 3 and result["jobs"][0]["attempt"] == 1, "publication crash duplicated the effect")
|
|
return {"broker_confirm_before_sql_commit": before, "confirmed_job_before_broker_restart": confirmed,
|
|
"message_while_publisher_held": {"id": linked["json"]["id"], "status": linked["status"],
|
|
"seconds": round(linked["elapsed"], 3)},
|
|
"broker_messages_before_restart": copies, "broker_messages_after_restart": restored, "after": result}
|
|
|
|
def worker_crash(self, phase, reference):
|
|
aid = self.queued()
|
|
marker = aid + "-v1-" + phase
|
|
self.arm(marker, [self.worker])
|
|
self.start("broker")
|
|
self.hit(marker, [self.worker])
|
|
before = self.state(aid)
|
|
check(before["variant_count"] == (3 if phase == "committed" else 0), "crash occurred on the wrong side of SQL commit")
|
|
observed = {name: self.object_hash(f"variants/{aid}/1/{name}/{value}") for name,value in reference.items()}
|
|
expected = {name:None for name in VARIANTS} if phase == "claimed" else reference
|
|
check(observed == expected, "crash occurred on the wrong side of storage writes")
|
|
if phase == "committed":
|
|
duplicate = aid + "-v1-duplicate"
|
|
self.arm(duplicate, [self.worker])
|
|
self.crash(self.worker)
|
|
self.marker(self.worker, marker, "release")
|
|
if phase == "committed":
|
|
self.hit(duplicate, [self.worker])
|
|
self.marker(self.worker, duplicate, "release")
|
|
result = self.result(aid)
|
|
want = 1 if phase == "committed" else 2
|
|
check(result["jobs"][0]["attempt"] == want and result["variant_count"] == 3, "incorrect crash recovery")
|
|
return {"phase": phase, "before": before, "storage_sha256_at_barrier": observed, "after": result}
|
|
|
|
def duplicate_ack(self, aid, version):
|
|
prefix = f"{aid}-v{version}-"
|
|
for phase in ["duplicate", "acked"]:
|
|
self.arm(prefix + phase, [self.worker])
|
|
self.cli("duplicate", aid, str(version), "1")
|
|
for phase in ["duplicate", "acked"]:
|
|
self.hit(prefix + phase, [self.worker])
|
|
self.marker(self.worker, prefix + phase, "release")
|
|
|
|
def lease_takeover(self):
|
|
aid = self.queued()
|
|
objects = aid + "-v1-objects"
|
|
committed = aid + "-v1-committed"
|
|
acked = aid + "-v1-acked"
|
|
self.arm(objects, [self.worker])
|
|
self.arm(committed, [self.worker])
|
|
self.start("broker")
|
|
self.hit(objects, [self.worker])
|
|
first = self.state(aid)
|
|
self.hit(committed, [self.worker])
|
|
winner = self.result(aid)
|
|
check(first["jobs"][0]["attempt"] == 1 and winner["jobs"][0]["attempt"] == 2
|
|
and first["jobs"][0]["id"] == winner["jobs"][0]["id"], "lease takeover changed logical work")
|
|
for barrier in [committed, objects]:
|
|
self.arm(acked, [self.worker])
|
|
self.marker(self.worker, barrier, "release")
|
|
self.hit(acked, [self.worker])
|
|
self.marker(self.worker, acked, "release")
|
|
after = self.result(aid)
|
|
check(after == winner and after["variant_count"] == 3, "expired worker modified published result")
|
|
return {"first_held": first, "second_committed": winner, "after_first_ack": after}
|
|
|
|
def reordered(self, v1_hashes):
|
|
aid = self.queued()
|
|
marker = aid + "-v1-objects"
|
|
self.arm(marker, [self.worker])
|
|
self.start("broker")
|
|
self.hit(marker, [self.worker])
|
|
self.duplicate_ack(aid, 1)
|
|
self.cli("reprocess", aid, "2")
|
|
before = self.result(aid)
|
|
check(before["published"] == 2 and before["variant_count"] == 3 and before["jobs"][0]["state"] == "superseded"
|
|
and before["jobs"][0]["attempt"] == 1,
|
|
"new version did not finish while first worker held its result")
|
|
check(v1_hashes["optimized"] != before["variant_sha256"]["optimized"], "fixture must distinguish the profiles")
|
|
acked = aid + "-v1-acked"
|
|
self.arm(acked, [self.worker])
|
|
self.marker(self.worker, marker, "release")
|
|
self.hit(acked, [self.worker])
|
|
self.marker(self.worker, acked, "release")
|
|
for version in [2, 1]:
|
|
self.duplicate_ack(aid, version)
|
|
after = self.result(aid)
|
|
check(after == before, "stale commit or delivery changed new result")
|
|
return {"reference_v1_webp": v1_hashes["optimized"], "before_releasing_v1": before,
|
|
"after_acks_in_order": [2, 1], "after": after}
|
|
|
|
def retry_replay(self):
|
|
aid = self.queued()
|
|
real = "staging/" + aid
|
|
self.sql(f"UPDATE attachment_jobs SET source_key='staging/{uuid.uuid4()}' WHERE attachment_id='{aid}'")
|
|
self.start("broker")
|
|
wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "retry exhaustion", 180)
|
|
failed = self.state(aid)
|
|
limit = int(self.exe(self.worker, "printenv", "JOB_MAX_ATTEMPTS"))
|
|
job = failed["jobs"][0]
|
|
check(job["attempt"] == limit and job["error"] == "source_missing", "wrong failure budget")
|
|
entries = json.loads(self.cli("failed"))
|
|
check(any(row["job_id"] == job["id"] for row in entries), "missing failed job")
|
|
self.sql(f"UPDATE attachment_jobs SET source_key='{real}' WHERE attachment_id='{aid}'")
|
|
self.cli("replay", aid, "track2")
|
|
result = self.result(aid)
|
|
check(result["jobs"][0]["attempt"] == limit + 1, "replay reused a stale attempt")
|
|
rejected = self.cli("replay", aid, "track2", allow_failure=True)
|
|
check(rejected.returncode != 0, "accepted replay of successful job")
|
|
count = int(self.sql(f"SELECT count(*) FROM attachment_replays WHERE job_id='{job['id']}' AND actor='track2'"))
|
|
check(count == 1, "incorrect audit count")
|
|
return {"injection": "absent source key assigned only to synthetic job", "exhausted": failed, "after": result, "audit_rows": count}
|
|
|
|
def permanent_failure(self, other_sha):
|
|
self.stop("broker")
|
|
aid = self.upload(fixture(19))
|
|
self.sql(f"UPDATE attachment_jobs SET source_key='blobs/{other_sha}' WHERE attachment_id='{aid}'")
|
|
self.start("broker")
|
|
wait(lambda: self.state(aid)["jobs"][0]["state"] == "failed", "permanent source mismatch")
|
|
failed = self.state(aid)
|
|
j = failed["jobs"][0]
|
|
check(j["attempt"] == 1 and j["error"] == "source_mismatch" and failed["variant_count"] == 0,
|
|
"permanent failure consumed additional attempts")
|
|
self.sql(f"UPDATE attachment_jobs SET source_key='staging/{aid}' WHERE attachment_id='{aid}'")
|
|
return {"injection": "different valid original assigned only to synthetic job", "failed": failed}
|
|
|
|
def cleanup_stage(self, aid, created):
|
|
# Run normal GC after its real minimum grace. No object timestamps are forged.
|
|
remaining = created + 125 - time.monotonic()
|
|
if remaining > 0:
|
|
time.sleep(remaining)
|
|
def run_gc(apply):
|
|
args = ["run", "--rm", "--no-deps", "-T", "reconcile", "--grace=2m"]
|
|
if apply:
|
|
args.append("--apply")
|
|
return [json.loads(line) for line in self.dc(*args, timeout=180).splitlines()]
|
|
def matches(rows):
|
|
return [row for row in rows if row.get("key") == "staging/" + aid]
|
|
before = matches(run_gc(False))
|
|
check(len(before) == 1 and before[0]["kind"] == "stale_staging", "unaccepted stage not reported")
|
|
applied = matches(run_gc(True))
|
|
check(len(applied) == 1 and applied[0]["applied"], "unaccepted stage not removed")
|
|
check(not matches(run_gc(False)), "removed stage still listed")
|
|
return {"before": before[0], "applied": applied[0], "after_findings": 0}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--base-url", required=True)
|
|
# Match the project selected for the Compose startup, including isolated review runs.
|
|
parser.add_argument("--project", choices=[os.environ.get("COMPOSE_PROJECT_NAME") or "lab5"], required=True)
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
exp = Experiment(args.base_url, args.project)
|
|
started = time.monotonic()
|
|
report = {"project": args.project, "complete": False, "scenarios": {}, "seconds": {},
|
|
"started_at":datetime.now(timezone.utc).isoformat()}
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
def save():
|
|
args.output.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
|
def record(name, callback):
|
|
print(name, flush=True)
|
|
beginning = time.monotonic()
|
|
report["scenarios"][name] = callback()
|
|
report["seconds"][name] = round(time.monotonic() - beginning, 3)
|
|
save()
|
|
return report["scenarios"][name]
|
|
save()
|
|
try:
|
|
stage = record("staging_before_commit", exp.staged_crash)
|
|
created = time.monotonic()
|
|
accepted = record("commit_before_publish", exp.outbox_restart)["after"]
|
|
record("broker_confirm_before_sql", exp.published_crash)
|
|
for phase in ["claimed", "objects", "committed"]:
|
|
record("worker_crash_" + phase, lambda: exp.worker_crash(phase, accepted["variant_sha256"]))
|
|
record("same_version_lease_takeover", exp.lease_takeover)
|
|
record("parallel_duplicate_and_reorder", lambda: exp.reordered(accepted["variant_sha256"]))
|
|
record("retry_exhaustion_replay", exp.retry_replay)
|
|
record("permanent_failure", lambda: exp.permanent_failure(accepted["original_sha256"]))
|
|
record("unaccepted_staging_cleanup", lambda: exp.cleanup_stage(stage["unaccepted"], created))
|
|
report["complete"] = True
|
|
report["finished_at"] = datetime.now(timezone.utc).isoformat()
|
|
report["elapsed_seconds"] = round(time.monotonic() - started, 3)
|
|
save()
|
|
finally:
|
|
for service in sorted(exp.stopped):
|
|
exp.start(service)
|
|
for container, stem in list(exp.armed):
|
|
exp.marker(container, stem, "release")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|