forked from mtusi2026/mtusi2026lab5
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> audit: БД только READ ONLY-транзакциями, S3 ключом auditor (Read+List): потерянная работа, один эффект на версию, опубликованные объекты и SHA-256, монотонная версия, источник у активной работы, посторонние объекты, попытки. pipeline_faults: дубль, reorder, crash relay после confirm, crash worker до/после побочного эффекта и после COMMIT, временные ошибки, два worker, устаревшая версия (v2 медленная, v3 быстрая; дубль v1), reorder версий в очереди, зависший worker (docker pause).
310 lines
17 KiB
Python
310 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Track 2 experiment: proven idempotency of the versioned pipeline under injected faults.
|
|
|
|
Hypothesis. With at-least-once delivery, duplicated and reordered messages, crashes of the worker
|
|
before/after its side effects and stale jobs of an older pipeline version:
|
|
H1 no accepted (202) work is lost - every upload ends ready (or failed with a code, never stuck);
|
|
H2 every effect (attachment, pipeline_version, variant) is published exactly once - one set of
|
|
variant rows written by ONE successful attempt, one set of objects in the bucket;
|
|
H3 the published version only grows - an older job never overwrites a newer result, `ready`
|
|
never goes back to queued/processing;
|
|
H4 objects of interrupted attempts are garbage that the cleanup removes; nothing published is
|
|
ever deleted.
|
|
|
|
Method. For every scenario the affected service (relay/worker) is recreated with FAULT_POINTS of
|
|
the scenario (the real code paths, real crashes = os._exit of the process, real `docker pause`),
|
|
N uploads go through the LB, then the scenario-specific facts are read from PostgreSQL and S3 and
|
|
the read-only invariant audit (messenger.audit, I1-I7, with SHA-256 of every published object)
|
|
runs over the WHOLE stand. The normal configuration is restored after each scenario.
|
|
|
|
python bench/pipeline_faults.py --output evidence/track2 [--uploads 5]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import platform
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "student-tests"))
|
|
import imagegen # noqa: E402
|
|
from conftest import (Api, attachment_row, compose, events_of, jobctl, psql, s3_keys, service_with_env, # noqa: E402
|
|
wait_healthy)
|
|
|
|
FAST = {"JOB_LEASE_SECONDS": "6", "ORPHAN_GRACE_SECONDS": "3", "CLEANUP_INTERVAL_SECONDS": "1"}
|
|
|
|
|
|
def now() -> str:
|
|
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
|
|
|
|
|
def wait(condition, timeout: float, what: str):
|
|
deadline = time.monotonic() + timeout
|
|
while True:
|
|
value = condition()
|
|
if value:
|
|
return value
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError(what)
|
|
time.sleep(0.5)
|
|
|
|
|
|
def facts(attachment_id: str, objects: set[str]) -> dict:
|
|
row = attachment_row(attachment_id)
|
|
out = psql("SELECT json_build_object("
|
|
"'claims', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = '{0}'), "
|
|
"'succeeded_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
|
" WHERE j.attachment_id = '{0}' AND a.outcome = 'succeeded'), "
|
|
"'variant_rows', (SELECT count(*) FROM attachment_variants WHERE attachment_id = '{0}'), "
|
|
"'versions', (SELECT coalesce(json_agg(DISTINCT pipeline_version), '[]') FROM attachment_variants "
|
|
" WHERE attachment_id = '{0}'), "
|
|
"'job_states', (SELECT coalesce(json_agg(state || ':v' || pipeline_version ORDER BY created_at), '[]') "
|
|
" FROM jobs WHERE attachment_id = '{0}'), "
|
|
"'uncleaned_failed_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
|
" WHERE j.attachment_id = '{0}' AND a.outcome <> 'succeeded' AND a.objects_cleaned_at IS NULL))"
|
|
.format(attachment_id))
|
|
f = json.loads(out)
|
|
f.update(status=row["status"], published_version=row["published_version"], staging_state=row["staging_state"],
|
|
objects=len([k for k in objects if k.startswith(f"a/{attachment_id}/")]))
|
|
f["events"] = [e["event"] for e in events_of(attachment_id)]
|
|
return f
|
|
|
|
|
|
def audit() -> dict:
|
|
r = compose("run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums", check=False, timeout=600)
|
|
report = json.loads(r.stdout)
|
|
return {"ok": report["ok"], "violations": report["violations"], "notes": len(report["notes"]),
|
|
"objects": report["objects"]}
|
|
|
|
|
|
class Run:
|
|
def __init__(self, api: Api, uploads: int):
|
|
self.api = api
|
|
self.n = uploads
|
|
self.alice, self.bob = api.user("t2a"), api.user("t2b")
|
|
self.chat = api.chat(self.alice, self.bob)
|
|
|
|
def upload_many(self, n: int | None = None) -> list[dict]:
|
|
return [self.api.upload(self.alice, self.chat, imagegen.png(60 + i, 40, noise=True)).json()
|
|
for i in range(n or self.n)]
|
|
|
|
def finish(self, atts: list[dict], timeout: float = 180) -> tuple[list[dict], list[float]]:
|
|
started = time.monotonic()
|
|
took = []
|
|
for a in atts:
|
|
self.api.wait_status(self.bob, a["id"], timeout=timeout)
|
|
took.append(round(time.monotonic() - started, 2))
|
|
return atts, took
|
|
|
|
|
|
def expect_one_effect(f: dict, version: int = 1) -> list[str]:
|
|
"""H1 + H2 for a plain upload of pipeline `version`."""
|
|
problems = []
|
|
if f["status"] != "ready":
|
|
problems.append(f"status {f['status']}")
|
|
if f["succeeded_attempts"] != 1 or f["variant_rows"] != 3 or f["published_version"] != version:
|
|
problems.append(f"effect {f['succeeded_attempts']} attempts/{f['variant_rows']} rows/v{f['published_version']}")
|
|
if f["objects"] != 4: # original + thumbnail + crop + optimized, cleanup done
|
|
problems.append(f"{f['objects']} objects in the bucket")
|
|
return problems
|
|
|
|
|
|
def plain_scenario(run: Run, name: str, service: str, env: dict, *, scale: int | None = None,
|
|
cleanup_wait: bool = True) -> dict:
|
|
with service_with_env(service, scale=scale, **env):
|
|
atts, took = run.finish(run.upload_many())
|
|
if cleanup_wait: # garbage of interrupted attempts: cleaned after ORPHAN_GRACE_SECONDS
|
|
try:
|
|
wait(lambda: all(json.loads(psql(
|
|
"SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
|
f"WHERE j.attachment_id = '{a['id']}' AND a.outcome <> 'succeeded' "
|
|
"AND a.objects_cleaned_at IS NULL"))["n"] == 0 for a in atts), 60, "cleanup")
|
|
except AssertionError:
|
|
pass # reported below as uncleaned attempts
|
|
objects = s3_keys("a/")
|
|
per = [facts(a["id"], objects) for a in atts]
|
|
problems = {a["id"]: p for a, f in zip(atts, per) if (p := expect_one_effect(f))}
|
|
return {"scenario": name, "faults": {service: env}, "uploads": len(atts),
|
|
"ready": sum(f["status"] == "ready" for f in per), "claims_total": sum(f["claims"] for f in per),
|
|
"max_claims": max(f["claims"] for f in per), "time_to_ready_s": {"p50": statistics.median(took),
|
|
"max": max(took)},
|
|
"event_sequences": sorted({" > ".join(f["events"]) for f in per}), "problems": problems}
|
|
|
|
|
|
def versions_scenario(run: Run) -> dict:
|
|
"""Stale job after a newer version: v2 (slow) and v3 run at the same time; v3 publishes first,
|
|
v2 finishes later and must end superseded without touching the published v3 result."""
|
|
att = run.upload_many(1)[0]
|
|
run.api.wait_status(run.bob, att["id"], "ready")
|
|
observed = set()
|
|
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10,slow_pipeline=2"):
|
|
jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "new encoder")
|
|
wait(lambda: "processing:v2" in facts(att["id"], set())["job_states"], 60, "v2 not claimed")
|
|
jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "newer encoder")
|
|
deadline = time.monotonic() + 120
|
|
while time.monotonic() < deadline:
|
|
meta = run.api.attachment(run.bob, att["id"])
|
|
observed.add(meta["status"])
|
|
states = facts(att["id"], set())["job_states"]
|
|
if not any(s.startswith(("queued", "processing")) for s in states):
|
|
break
|
|
time.sleep(0.3)
|
|
final = run.api.attachment(run.bob, att["id"])
|
|
f = facts(att["id"], s3_keys("a/"))
|
|
thumb_v3 = psql(f"SELECT encode(sha256, 'hex') FROM attachment_variants WHERE attachment_id = '{att['id']}' "
|
|
"AND pipeline_version = 3 AND name = 'thumbnail'")
|
|
problems = []
|
|
if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"]:
|
|
problems.append(f"job states {f['job_states']}")
|
|
if f["published_version"] != 3 or observed != {"ready"}:
|
|
problems.append(f"published v{f['published_version']}, statuses seen {sorted(observed)}")
|
|
if next(v["sha256"] for v in final["variants"] if v["name"] == "thumbnail") != thumb_v3:
|
|
problems.append("published thumbnail is not the v3 one")
|
|
# Stale duplicate of the FIRST job, published by hand with the relay's broker user: a no-op.
|
|
job_v1 = psql(f"SELECT id FROM jobs WHERE attachment_id = '{att['id']}' AND pipeline_version = 1")
|
|
script = ("import asyncio, aio_pika, datetime as d\n"
|
|
"from messenger import broker\n"
|
|
"async def m():\n"
|
|
" c = await aio_pika.connect(open('/run/secrets/broker_url').read().strip())\n"
|
|
" ch = await c.channel(publisher_confirms=True)\n"
|
|
" ex = await ch.get_exchange(broker.EXCHANGE, ensure=False)\n"
|
|
f" body = broker.encode('{job_v1}', '{att['id']}', 1, 1, d.datetime.now(d.timezone.utc))\n"
|
|
" await ex.publish(aio_pika.Message(body), routing_key=broker.ROUTING_KEY)\n"
|
|
" await c.close()\n"
|
|
"asyncio.run(m())\n")
|
|
compose("exec", "-T", "relay", "python", "-c", script)
|
|
time.sleep(3)
|
|
after = run.api.attachment(run.bob, att["id"])
|
|
if after != final:
|
|
problems.append("stale v1 message changed the attachment")
|
|
return {"scenario": "stale_version_and_reorder", "uploads": 1, "statuses_seen_during_reprocess": sorted(observed),
|
|
"job_states": f["job_states"], "published_version": f["published_version"],
|
|
"variant_rows_by_version": f["variant_rows"], "objects": f["objects"],
|
|
"events": f["events"], "problems": problems}
|
|
|
|
|
|
def queued_reorder_scenario(run: Run) -> dict:
|
|
"""Both newer versions wait in the outbox; the relay publishes them in REVERSE order and one worker
|
|
takes them one by one: v3 is published first, v2 is then superseded at its claim (no work)."""
|
|
att = run.upload_many(1)[0]
|
|
run.api.wait_status(run.bob, att["id"], "ready")
|
|
# The worker stays stopped until it is recreated with concurrency 1 (no normal start in between).
|
|
# Worker and relay stay stopped until each is recreated with the scenario's settings: no normal
|
|
# start in between that would consume/publish the jobs in the ordinary order.
|
|
compose("stop", "worker", "relay")
|
|
try:
|
|
jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "v2")
|
|
jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "v3")
|
|
with service_with_env("relay", RELAY_FAULT_POINTS="reverse_order"):
|
|
wait(lambda: psql("SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = "
|
|
f"'{att['id']}' AND j.pipeline_version > 1 AND o.published_at IS NOT NULL") == "2",
|
|
30, "both reprocess jobs not published") # in one relay batch: v3 first, then v2
|
|
with service_with_env("worker", WORKER_CONCURRENCY="1"):
|
|
wait(lambda: not any(s.startswith(("queued", "processing"))
|
|
for s in facts(att["id"], set())["job_states"]), 120, "reprocess jobs did not finish")
|
|
finally:
|
|
compose("start", "worker", "relay")
|
|
wait_healthy("worker", "relay")
|
|
f = facts(att["id"], s3_keys("a/"))
|
|
problems = []
|
|
if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"] or f["published_version"] != 3:
|
|
problems.append(f"{f['job_states']} published v{f['published_version']}")
|
|
if f["claims"] != 2: # v1 and v3 only: v2 was superseded before any work
|
|
problems.append(f"{f['claims']} claims")
|
|
return {"scenario": "reordered_versions_in_queue", "job_states": f["job_states"], "claims": f["claims"],
|
|
"published_version": f["published_version"], "events": f["events"], "problems": problems}
|
|
|
|
|
|
def hung_worker_scenario(run: Run) -> dict:
|
|
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST):
|
|
att = run.upload_many(1)[0]
|
|
owner = wait(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
|
|
"AND state = 'processing'"), 30, "not claimed")
|
|
ids = compose("ps", "-q", "worker").stdout.split()
|
|
names = dict(line.split() for line in subprocess.run(
|
|
["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True, text=True,
|
|
check=True).stdout.splitlines())
|
|
subprocess.run(["docker", "pause", names[owner]], check=True)
|
|
try:
|
|
run.api.wait_status(run.bob, att["id"], "ready", timeout=150)
|
|
published = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants "
|
|
f"WHERE attachment_id = '{att['id']}'")
|
|
finally:
|
|
subprocess.run(["docker", "unpause", names[owner]], check=True)
|
|
time.sleep(15)
|
|
again = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants "
|
|
f"WHERE attachment_id = '{att['id']}'")
|
|
wait(lambda: json.loads(psql("SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j "
|
|
f"ON j.id = a.job_id WHERE j.attachment_id = '{att['id']}' AND a.outcome <> "
|
|
"'succeeded' AND a.objects_cleaned_at IS NULL"))["n"] == 0, 60, "cleanup")
|
|
f = facts(att["id"], s3_keys("a/"))
|
|
problems = expect_one_effect(f)
|
|
if published != again:
|
|
problems.append("the resumed (fenced) attempt changed the published result")
|
|
return {"scenario": "hung_worker_docker_pause", "frozen": owner, "claims": f["claims"], "events": f["events"],
|
|
"problems": problems}
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--output", default="evidence/track2")
|
|
p.add_argument("--uploads", type=int, default=5)
|
|
args = p.parse_args()
|
|
out = ROOT / args.output
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
api = Api()
|
|
run = Run(api, args.uploads)
|
|
started = now()
|
|
results = []
|
|
plan = [
|
|
("baseline_no_faults", "worker", {}),
|
|
("duplicate_publish", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish"}),
|
|
("duplicate_and_reverse_order", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish,reverse_order"}),
|
|
("relay_crash_after_confirm_before_mark", "relay", {"RELAY_FAULT_POINTS": "crash_after_publish"}),
|
|
("worker_crash_before_side_effect", "worker", {"WORKER_FAULT_POINTS": "crash_after_claim", **FAST}),
|
|
("worker_crash_after_s3_before_commit", "worker", {"WORKER_FAULT_POINTS": "crash_after_s3_write", **FAST}),
|
|
("worker_crash_after_commit_before_ack", "worker", {"WORKER_FAULT_POINTS": "crash_after_commit", **FAST}),
|
|
("transient_errors_then_success", "worker", {"WORKER_FAULT_POINTS": "transient_error=2",
|
|
"JOB_RETRY_BASE_SECONDS": "0.5", **FAST}),
|
|
]
|
|
for name, service, env in plan:
|
|
print(f"[{now()}] {name}", flush=True)
|
|
results.append(plain_scenario(run, name, service, env))
|
|
print(f"[{now()}] two_workers_duplicate_publish", flush=True)
|
|
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish"):
|
|
results.append(plain_scenario(run, "two_workers_duplicate_publish", "worker", {}, scale=2))
|
|
for scenario in (versions_scenario, queued_reorder_scenario, hung_worker_scenario):
|
|
print(f"[{now()}] {scenario.__name__}", flush=True)
|
|
results.append(scenario(run))
|
|
wait_healthy("worker", "relay")
|
|
report = audit()
|
|
summary = {
|
|
"experiment": "track 2: idempotency of the versioned pipeline under injected faults",
|
|
"started": started, "finished": now(),
|
|
"host": {"os": platform.platform(), "python": platform.python_version()},
|
|
"uploads_per_plain_scenario": args.uploads,
|
|
"scenarios": results,
|
|
"audit_after_all": report,
|
|
"violations": sum(len(r["problems"]) for r in results) + len(report["violations"]),
|
|
}
|
|
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8")
|
|
lines = [f"{'scenario':45} {'uploads':>7} {'ready':>5} {'claims':>6} problems"]
|
|
for r in results:
|
|
lines.append(f"{r['scenario']:45} {r.get('uploads', 1):>7} {r.get('ready', '-'):>5} "
|
|
f"{r.get('claims_total', r.get('claims', '-')):>6} {len(r['problems'])}")
|
|
lines.append(f"audit after all scenarios: ok={report['ok']} violations={len(report['violations'])} "
|
|
f"objects={report['objects']}")
|
|
(out / "summary.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
print("\n".join(lines))
|
|
return 0 if summary["violations"] == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|