486 lines
15 KiB
Python
486 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
|
|
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
|
|
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
import pika
|
|
|
|
from src.publisher import publish_pending
|
|
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
|
|
|
|
|
|
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
|
|
JOB_ID = "00000000-0000-4000-8000-000000000002"
|
|
|
|
|
|
class SimulatedCrash(BaseException):
|
|
pass
|
|
|
|
|
|
class FakeObjects:
|
|
def __init__(self, content):
|
|
self.data = {"staging/" + ATTACHMENT_ID: content}
|
|
self.puts = []
|
|
|
|
def get(self, key, max_bytes):
|
|
return self.data[key]
|
|
|
|
def put(self, key, content, content_type, size_bytes=None):
|
|
value = content if isinstance(content, bytes) else content.read()
|
|
assert len(value) == size_bytes
|
|
self.data[key] = value
|
|
self.puts.append((key, hashlib.sha256(value).hexdigest()))
|
|
|
|
def delete(self, key):
|
|
self.data.pop(key, None)
|
|
|
|
|
|
class FakeStore:
|
|
def __init__(self, content):
|
|
self.objects = FakeObjects(content)
|
|
self.row = {
|
|
"id": ATTACHMENT_ID,
|
|
"job_id": JOB_ID,
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"status": "queued",
|
|
"lease_until": 0.0,
|
|
"staging_key": "staging/" + ATTACHMENT_ID,
|
|
"object_key": "original/" + ATTACHMENT_ID,
|
|
"content_type": "image/png",
|
|
"size_bytes": len(content),
|
|
"sha256": hashlib.sha256(content).hexdigest(),
|
|
"variants": [],
|
|
}
|
|
|
|
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
|
|
row = self.row
|
|
if (
|
|
attachment_id != row["id"]
|
|
or job_id != row["job_id"]
|
|
or version != row["pipeline_version"]
|
|
or attempt != row["attempt"]
|
|
or row["status"] in ("ready", "failed")
|
|
):
|
|
return None
|
|
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
|
|
return None
|
|
row["status"] = "processing"
|
|
row["lease_until"] = time.monotonic() + lease_seconds
|
|
return dict(row)
|
|
|
|
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
|
|
row = self.row
|
|
if (
|
|
attachment_id != row["id"]
|
|
or job_id != row["job_id"]
|
|
or version != row["pipeline_version"]
|
|
or attempt != row["attempt"]
|
|
or row["status"] != "processing"
|
|
or row["lease_until"] <= time.monotonic()
|
|
):
|
|
return False
|
|
row["status"] = "ready"
|
|
row["lease_until"] = 0.0
|
|
row["variants"] = variants
|
|
return True
|
|
|
|
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
|
|
row = self.row
|
|
if (
|
|
row["id"] == attachment_id
|
|
and row["job_id"] == job_id
|
|
and row["pipeline_version"] == version
|
|
and row["attempt"] == attempt
|
|
and row["status"] == "processing"
|
|
):
|
|
row["status"] = "failed" if delay is None else "queued"
|
|
row["lease_until"] = 0.0
|
|
row["error_code"] = error_code
|
|
if delay is not None:
|
|
row["attempt"] += 1
|
|
|
|
def recover_expired(self):
|
|
if self.row["status"] == "processing":
|
|
self.row["attempt"] += 1
|
|
self.row["status"] = "queued"
|
|
self.row["lease_until"] = 0.0
|
|
|
|
def replay(self, version):
|
|
row = self.row
|
|
previous_job = row["job_id"]
|
|
row["job_id"] = str(uuid.uuid4())
|
|
row["pipeline_version"] = version
|
|
row["attempt"] = 1
|
|
row["status"] = "queued"
|
|
row["lease_until"] = 0.0
|
|
row["variants"] = []
|
|
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
|
|
self.objects.put(
|
|
row["staging_key"],
|
|
self.objects.data[row["object_key"]],
|
|
row["content_type"],
|
|
row["size_bytes"],
|
|
)
|
|
return {
|
|
"schema_version": 1,
|
|
"job_id": row["job_id"],
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": ATTACHMENT_ID,
|
|
"pipeline_version": version,
|
|
"attempt": 1,
|
|
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
|
"previous_job_id": previous_job,
|
|
}
|
|
|
|
|
|
class FakeChannel:
|
|
def __init__(self):
|
|
self.acked = []
|
|
self.nacked = []
|
|
|
|
def basic_ack(self, delivery_tag):
|
|
self.acked.append(delivery_tag)
|
|
|
|
def basic_nack(self, delivery_tag, requeue):
|
|
self.nacked.append((delivery_tag, requeue))
|
|
|
|
|
|
class FakeMethod:
|
|
def __init__(self, delivery_tag):
|
|
self.delivery_tag = delivery_tag
|
|
|
|
|
|
def job_body(job_id=JOB_ID, version=1, attempt=1):
|
|
return json.dumps(
|
|
{
|
|
"schema_version": 1,
|
|
"job_id": job_id,
|
|
"type": "attachment.process.v1",
|
|
"attachment_id": ATTACHMENT_ID,
|
|
"pipeline_version": version,
|
|
"attempt": attempt,
|
|
"created_at": "2026-01-01T00:00:00Z",
|
|
}
|
|
).encode()
|
|
|
|
|
|
def crash_at(expected):
|
|
def inject(point):
|
|
if point == expected:
|
|
raise SimulatedCrash(point)
|
|
|
|
return inject
|
|
|
|
|
|
def slow_builder(attachment_id, content, pipeline_version):
|
|
time.sleep(5)
|
|
|
|
|
|
def test_duplicate_and_ack_loss(content):
|
|
store = FakeStore(content)
|
|
channel = FakeChannel()
|
|
try:
|
|
handle_delivery(
|
|
store, channel, FakeMethod(1), job_body(),
|
|
fault_injector=crash_at("before_ack"),
|
|
)
|
|
raise AssertionError("fault injection did not crash before ack")
|
|
except SimulatedCrash:
|
|
pass
|
|
assert store.row["status"] == "ready"
|
|
first_write_count = len(store.objects.puts)
|
|
assert channel.acked == []
|
|
|
|
handle_delivery(store, channel, FakeMethod(2), job_body())
|
|
assert channel.acked == [2]
|
|
assert len(store.objects.puts) == first_write_count
|
|
assert store.row["status"] == "ready"
|
|
return {
|
|
"scenario": "duplicate_delivery_after_commit_before_ack",
|
|
"deliveries": 2,
|
|
"effective_object_writes": first_write_count,
|
|
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
|
|
}
|
|
|
|
|
|
def test_crash_before_and_after_side_effect(content, crash_point):
|
|
store = FakeStore(content)
|
|
channel = FakeChannel()
|
|
try:
|
|
handle_delivery(
|
|
store, channel, FakeMethod(3), job_body(),
|
|
fault_injector=crash_at(crash_point),
|
|
)
|
|
raise AssertionError(f"fault injection did not crash at {crash_point}")
|
|
except SimulatedCrash:
|
|
pass
|
|
assert channel.acked == []
|
|
store.recover_expired()
|
|
retry = job_body(attempt=2)
|
|
handle_delivery(store, channel, FakeMethod(4), retry)
|
|
assert store.row["status"] == "ready"
|
|
assert store.row["attempt"] == 2
|
|
assert channel.acked == [4]
|
|
assert store.objects.data["original/" + ATTACHMENT_ID] == content
|
|
return {
|
|
"scenario": "crash_" + crash_point,
|
|
"attempts": store.row["attempt"],
|
|
"terminal_status": store.row["status"],
|
|
}
|
|
|
|
|
|
def test_reordered_attempts(content):
|
|
store = FakeStore(content)
|
|
store.row["attempt"] = 2
|
|
channel = FakeChannel()
|
|
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
|
|
writes_after_current = len(store.objects.puts)
|
|
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
|
|
assert store.row["status"] == "ready"
|
|
assert len(store.objects.puts) == writes_after_current
|
|
assert channel.acked == [5, 6]
|
|
return {
|
|
"scenario": "out_of_order_attempt_delivery",
|
|
"current_attempt": store.row["attempt"],
|
|
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
|
|
}
|
|
|
|
|
|
def test_stale_pipeline_cannot_publish_newer_result(content):
|
|
store = FakeStore(content)
|
|
channel = FakeChannel()
|
|
replayed = []
|
|
|
|
def supersede_after_first_variant(point):
|
|
if point == "after_variant_put:thumbnail" and not replayed:
|
|
replayed.append(store.replay(2))
|
|
|
|
handle_delivery(
|
|
store, channel, FakeMethod(7), job_body(),
|
|
fault_injector=supersede_after_first_variant,
|
|
)
|
|
assert store.row["status"] == "queued"
|
|
assert store.row["pipeline_version"] == 2
|
|
assert channel.acked == [7]
|
|
|
|
current_job = replayed[0]
|
|
handle_delivery(
|
|
store,
|
|
channel,
|
|
FakeMethod(8),
|
|
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
|
|
)
|
|
assert store.row["status"] == "ready"
|
|
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
|
|
assert channel.acked == [7, 8]
|
|
return {
|
|
"scenario": "stale_pipeline_after_newer_version",
|
|
"current_pipeline_version": store.row["pipeline_version"],
|
|
"published_variant_versions": sorted(
|
|
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
|
|
),
|
|
}
|
|
|
|
|
|
class FakeTransaction:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
|
|
def __enter__(self):
|
|
self.published = False
|
|
self.store.current_transaction = self
|
|
return self
|
|
|
|
def __exit__(self, exc_type, exc_value, traceback):
|
|
if exc_type is None and self.published:
|
|
self.store.outbox_published = True
|
|
self.store.current_transaction = None
|
|
return False
|
|
|
|
|
|
class FakeConnection:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
|
|
def transaction(self):
|
|
return FakeTransaction(self.store)
|
|
|
|
def execute(self, query, params):
|
|
assert "UPDATE attachment_outbox" in query
|
|
assert params == ("outbox-row",)
|
|
self.store.current_transaction.published = True
|
|
|
|
|
|
class FakePool:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
|
|
def connection(self):
|
|
connection = FakeConnection(self.store)
|
|
|
|
class ConnectionContext:
|
|
def __enter__(self_inner):
|
|
return connection
|
|
|
|
def __exit__(self_inner, exc_type, exc_value, traceback):
|
|
return False
|
|
|
|
return ConnectionContext()
|
|
|
|
|
|
class FakeOutboxStore:
|
|
def __init__(self):
|
|
self.pool = FakePool(self)
|
|
self.outbox_published = False
|
|
self.current_transaction = None
|
|
|
|
def claim_outbox(self, connection):
|
|
if self.outbox_published:
|
|
return None
|
|
return {
|
|
"id": "outbox-row",
|
|
"job_id": JOB_ID,
|
|
"attachment_id": ATTACHMENT_ID,
|
|
"pipeline_version": 1,
|
|
"attempt": 1,
|
|
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
|
|
}
|
|
|
|
|
|
class AmbiguousConfirmChannel:
|
|
def __init__(self):
|
|
self.messages = []
|
|
self.fail_once = True
|
|
|
|
def basic_publish(self, **message):
|
|
self.messages.append(message)
|
|
if self.fail_once:
|
|
self.fail_once = False
|
|
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
|
|
|
|
|
|
class FailBeforePublishChannel:
|
|
def __init__(self):
|
|
self.messages = []
|
|
self.fail_once = True
|
|
|
|
def basic_publish(self, **message):
|
|
if self.fail_once:
|
|
self.fail_once = False
|
|
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
|
|
self.messages.append(message)
|
|
|
|
|
|
def test_committed_outbox_survives_publisher_crash():
|
|
store = FakeOutboxStore()
|
|
channel = FailBeforePublishChannel()
|
|
try:
|
|
publish_pending(store, channel)
|
|
raise AssertionError("publisher crash was not injected")
|
|
except pika.exceptions.AMQPConnectionError:
|
|
pass
|
|
assert not store.outbox_published
|
|
assert channel.messages == []
|
|
assert publish_pending(store, channel)
|
|
assert store.outbox_published
|
|
assert len(channel.messages) == 1
|
|
return {
|
|
"scenario": "publisher_crash_before_publish",
|
|
"published_after_restart": len(channel.messages),
|
|
"outbox_recovered_after_restart": store.outbox_published,
|
|
}
|
|
|
|
|
|
def test_outbox_confirm_gap_republishes_same_job():
|
|
store = FakeOutboxStore()
|
|
channel = AmbiguousConfirmChannel()
|
|
try:
|
|
publish_pending(store, channel)
|
|
raise AssertionError("publisher confirm fault was not injected")
|
|
except pika.exceptions.AMQPConnectionError:
|
|
pass
|
|
assert not store.outbox_published
|
|
assert publish_pending(store, channel)
|
|
assert store.outbox_published
|
|
assert len(channel.messages) == 2
|
|
first, second = channel.messages
|
|
assert first["body"] == second["body"]
|
|
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
|
|
return {
|
|
"scenario": "publish_confirm_lost_after_broker_accept",
|
|
"deliveries": len(channel.messages),
|
|
"same_message_id": True,
|
|
}
|
|
|
|
|
|
def test_image_processing_has_a_hard_timeout():
|
|
started = time.monotonic()
|
|
try:
|
|
_build_variants_bounded(
|
|
ATTACHMENT_ID,
|
|
b"bounded timeout test",
|
|
1,
|
|
timeout_seconds=1,
|
|
builder=slow_builder,
|
|
)
|
|
raise AssertionError("slow transform exceeded its time budget without termination")
|
|
except ProcessingTimeout:
|
|
pass
|
|
elapsed = time.monotonic() - started
|
|
assert elapsed < 4
|
|
return {
|
|
"scenario": "transform_timeout_terminated",
|
|
"configured_timeout_seconds": 1,
|
|
"observed_seconds": round(elapsed, 3),
|
|
}
|
|
|
|
|
|
def main():
|
|
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
results = [
|
|
test_duplicate_and_ack_loss(content),
|
|
test_crash_before_and_after_side_effect(content, "after_claim"),
|
|
test_crash_before_and_after_side_effect(content, "after_original_put"),
|
|
test_reordered_attempts(content),
|
|
test_stale_pipeline_cannot_publish_newer_result(content),
|
|
test_committed_outbox_survives_publisher_crash(),
|
|
test_outbox_confirm_gap_republishes_same_job(),
|
|
test_image_processing_has_a_hard_timeout(),
|
|
]
|
|
report = {
|
|
"result": "PASS",
|
|
"track": "2",
|
|
"fixture": "tests/fixtures/sample.png",
|
|
"scenarios": results,
|
|
"invariants": {
|
|
"confirmed_work_eventually_ready": True,
|
|
"duplicate_effects_create_no_second_variant_set": True,
|
|
"stale_attempt_writes_no_effects": True,
|
|
"stale_pipeline_cannot_publish_current_metadata": True,
|
|
"ambiguous_confirm_republishes_stable_job": True,
|
|
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
|
|
},
|
|
}
|
|
evidence = ROOT / "evidence" / "pipeline-faults.json"
|
|
evidence.parent.mkdir(exist_ok=True)
|
|
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
print(json.dumps(report, sort_keys=True))
|
|
print(f"Evidence written to {evidence.relative_to(ROOT)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|