Files
mtusi2026lab7/student-tests/test_pipeline_faults.py
T
kaktuz20000 5a6fcff3ef
Skeleton contracts / contracts (pull_request) Canceled after 0s
выполение на 5
2026-11-04 19:22:35 +03:00

486 lines
15 KiB
Python

#!/usr/bin/env python3
"""Deterministic fault-injection checks for delivery, side effects, and pipeline fencing."""
import hashlib
import json
import os
from pathlib import Path
import sys
import time
import uuid
from datetime import datetime, timezone
os.environ.setdefault("S3_ACCESS_KEY_ID", "pipeline-test")
os.environ.setdefault("S3_SECRET_ACCESS_KEY", "pipeline-test-secret")
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
import pika
from src.publisher import publish_pending
from src.worker import ProcessingTimeout, _build_variants_bounded, handle_delivery
ATTACHMENT_ID = "00000000-0000-4000-8000-000000000001"
JOB_ID = "00000000-0000-4000-8000-000000000002"
class SimulatedCrash(BaseException):
pass
class FakeObjects:
def __init__(self, content):
self.data = {"staging/" + ATTACHMENT_ID: content}
self.puts = []
def get(self, key, max_bytes):
return self.data[key]
def put(self, key, content, content_type, size_bytes=None):
value = content if isinstance(content, bytes) else content.read()
assert len(value) == size_bytes
self.data[key] = value
self.puts.append((key, hashlib.sha256(value).hexdigest()))
def delete(self, key):
self.data.pop(key, None)
class FakeStore:
def __init__(self, content):
self.objects = FakeObjects(content)
self.row = {
"id": ATTACHMENT_ID,
"job_id": JOB_ID,
"pipeline_version": 1,
"attempt": 1,
"status": "queued",
"lease_until": 0.0,
"staging_key": "staging/" + ATTACHMENT_ID,
"object_key": "original/" + ATTACHMENT_ID,
"content_type": "image/png",
"size_bytes": len(content),
"sha256": hashlib.sha256(content).hexdigest(),
"variants": [],
}
def claim_attachment_job(self, attachment_id, job_id, version, attempt, lease_seconds):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] in ("ready", "failed")
):
return None
if row["status"] == "processing" and row["lease_until"] > time.monotonic():
return None
row["status"] = "processing"
row["lease_until"] = time.monotonic() + lease_seconds
return dict(row)
def complete_attachment_job(self, attachment_id, job_id, version, attempt, variants):
row = self.row
if (
attachment_id != row["id"]
or job_id != row["job_id"]
or version != row["pipeline_version"]
or attempt != row["attempt"]
or row["status"] != "processing"
or row["lease_until"] <= time.monotonic()
):
return False
row["status"] = "ready"
row["lease_until"] = 0.0
row["variants"] = variants
return True
def fail_attachment_job(self, attachment_id, job_id, version, attempt, max_attempts, error_code, delay):
row = self.row
if (
row["id"] == attachment_id
and row["job_id"] == job_id
and row["pipeline_version"] == version
and row["attempt"] == attempt
and row["status"] == "processing"
):
row["status"] = "failed" if delay is None else "queued"
row["lease_until"] = 0.0
row["error_code"] = error_code
if delay is not None:
row["attempt"] += 1
def recover_expired(self):
if self.row["status"] == "processing":
self.row["attempt"] += 1
self.row["status"] = "queued"
self.row["lease_until"] = 0.0
def replay(self, version):
row = self.row
previous_job = row["job_id"]
row["job_id"] = str(uuid.uuid4())
row["pipeline_version"] = version
row["attempt"] = 1
row["status"] = "queued"
row["lease_until"] = 0.0
row["variants"] = []
row["staging_key"] = f"staging/{ATTACHMENT_ID}/{row['job_id']}"
self.objects.put(
row["staging_key"],
self.objects.data[row["object_key"]],
row["content_type"],
row["size_bytes"],
)
return {
"schema_version": 1,
"job_id": row["job_id"],
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
"previous_job_id": previous_job,
}
class FakeChannel:
def __init__(self):
self.acked = []
self.nacked = []
def basic_ack(self, delivery_tag):
self.acked.append(delivery_tag)
def basic_nack(self, delivery_tag, requeue):
self.nacked.append((delivery_tag, requeue))
class FakeMethod:
def __init__(self, delivery_tag):
self.delivery_tag = delivery_tag
def job_body(job_id=JOB_ID, version=1, attempt=1):
return json.dumps(
{
"schema_version": 1,
"job_id": job_id,
"type": "attachment.process.v1",
"attachment_id": ATTACHMENT_ID,
"pipeline_version": version,
"attempt": attempt,
"created_at": "2026-01-01T00:00:00Z",
}
).encode()
def crash_at(expected):
def inject(point):
if point == expected:
raise SimulatedCrash(point)
return inject
def slow_builder(attachment_id, content, pipeline_version):
time.sleep(5)
def test_duplicate_and_ack_loss(content):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(1), job_body(),
fault_injector=crash_at("before_ack"),
)
raise AssertionError("fault injection did not crash before ack")
except SimulatedCrash:
pass
assert store.row["status"] == "ready"
first_write_count = len(store.objects.puts)
assert channel.acked == []
handle_delivery(store, channel, FakeMethod(2), job_body())
assert channel.acked == [2]
assert len(store.objects.puts) == first_write_count
assert store.row["status"] == "ready"
return {
"scenario": "duplicate_delivery_after_commit_before_ack",
"deliveries": 2,
"effective_object_writes": first_write_count,
"duplicate_delivery_writes": len(store.objects.puts) - first_write_count,
}
def test_crash_before_and_after_side_effect(content, crash_point):
store = FakeStore(content)
channel = FakeChannel()
try:
handle_delivery(
store, channel, FakeMethod(3), job_body(),
fault_injector=crash_at(crash_point),
)
raise AssertionError(f"fault injection did not crash at {crash_point}")
except SimulatedCrash:
pass
assert channel.acked == []
store.recover_expired()
retry = job_body(attempt=2)
handle_delivery(store, channel, FakeMethod(4), retry)
assert store.row["status"] == "ready"
assert store.row["attempt"] == 2
assert channel.acked == [4]
assert store.objects.data["original/" + ATTACHMENT_ID] == content
return {
"scenario": "crash_" + crash_point,
"attempts": store.row["attempt"],
"terminal_status": store.row["status"],
}
def test_reordered_attempts(content):
store = FakeStore(content)
store.row["attempt"] = 2
channel = FakeChannel()
handle_delivery(store, channel, FakeMethod(5), job_body(attempt=2))
writes_after_current = len(store.objects.puts)
handle_delivery(store, channel, FakeMethod(6), job_body(attempt=1))
assert store.row["status"] == "ready"
assert len(store.objects.puts) == writes_after_current
assert channel.acked == [5, 6]
return {
"scenario": "out_of_order_attempt_delivery",
"current_attempt": store.row["attempt"],
"stale_attempt_extra_writes": len(store.objects.puts) - writes_after_current,
}
def test_stale_pipeline_cannot_publish_newer_result(content):
store = FakeStore(content)
channel = FakeChannel()
replayed = []
def supersede_after_first_variant(point):
if point == "after_variant_put:thumbnail" and not replayed:
replayed.append(store.replay(2))
handle_delivery(
store, channel, FakeMethod(7), job_body(),
fault_injector=supersede_after_first_variant,
)
assert store.row["status"] == "queued"
assert store.row["pipeline_version"] == 2
assert channel.acked == [7]
current_job = replayed[0]
handle_delivery(
store,
channel,
FakeMethod(8),
job_body(current_job["job_id"], current_job["pipeline_version"], current_job["attempt"]),
)
assert store.row["status"] == "ready"
assert {variant["object_key"].split("/")[2] for variant in store.row["variants"]} == {"v2"}
assert channel.acked == [7, 8]
return {
"scenario": "stale_pipeline_after_newer_version",
"current_pipeline_version": store.row["pipeline_version"],
"published_variant_versions": sorted(
{variant["object_key"].split("/")[2] for variant in store.row["variants"]}
),
}
class FakeTransaction:
def __init__(self, store):
self.store = store
def __enter__(self):
self.published = False
self.store.current_transaction = self
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type is None and self.published:
self.store.outbox_published = True
self.store.current_transaction = None
return False
class FakeConnection:
def __init__(self, store):
self.store = store
def transaction(self):
return FakeTransaction(self.store)
def execute(self, query, params):
assert "UPDATE attachment_outbox" in query
assert params == ("outbox-row",)
self.store.current_transaction.published = True
class FakePool:
def __init__(self, store):
self.store = store
def connection(self):
connection = FakeConnection(self.store)
class ConnectionContext:
def __enter__(self_inner):
return connection
def __exit__(self_inner, exc_type, exc_value, traceback):
return False
return ConnectionContext()
class FakeOutboxStore:
def __init__(self):
self.pool = FakePool(self)
self.outbox_published = False
self.current_transaction = None
def claim_outbox(self, connection):
if self.outbox_published:
return None
return {
"id": "outbox-row",
"job_id": JOB_ID,
"attachment_id": ATTACHMENT_ID,
"pipeline_version": 1,
"attempt": 1,
"created_at": datetime(2026, 1, 1, tzinfo=timezone.utc),
}
class AmbiguousConfirmChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
self.messages.append(message)
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated lost publisher confirm")
class FailBeforePublishChannel:
def __init__(self):
self.messages = []
self.fail_once = True
def basic_publish(self, **message):
if self.fail_once:
self.fail_once = False
raise pika.exceptions.AMQPConnectionError("simulated publisher crash before publish")
self.messages.append(message)
def test_committed_outbox_survives_publisher_crash():
store = FakeOutboxStore()
channel = FailBeforePublishChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher crash was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert channel.messages == []
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 1
return {
"scenario": "publisher_crash_before_publish",
"published_after_restart": len(channel.messages),
"outbox_recovered_after_restart": store.outbox_published,
}
def test_outbox_confirm_gap_republishes_same_job():
store = FakeOutboxStore()
channel = AmbiguousConfirmChannel()
try:
publish_pending(store, channel)
raise AssertionError("publisher confirm fault was not injected")
except pika.exceptions.AMQPConnectionError:
pass
assert not store.outbox_published
assert publish_pending(store, channel)
assert store.outbox_published
assert len(channel.messages) == 2
first, second = channel.messages
assert first["body"] == second["body"]
assert first["properties"].message_id == second["properties"].message_id == "outbox-row"
return {
"scenario": "publish_confirm_lost_after_broker_accept",
"deliveries": len(channel.messages),
"same_message_id": True,
}
def test_image_processing_has_a_hard_timeout():
started = time.monotonic()
try:
_build_variants_bounded(
ATTACHMENT_ID,
b"bounded timeout test",
1,
timeout_seconds=1,
builder=slow_builder,
)
raise AssertionError("slow transform exceeded its time budget without termination")
except ProcessingTimeout:
pass
elapsed = time.monotonic() - started
assert elapsed < 4
return {
"scenario": "transform_timeout_terminated",
"configured_timeout_seconds": 1,
"observed_seconds": round(elapsed, 3),
}
def main():
content = (ROOT / "tests/fixtures/sample.png").read_bytes()
results = [
test_duplicate_and_ack_loss(content),
test_crash_before_and_after_side_effect(content, "after_claim"),
test_crash_before_and_after_side_effect(content, "after_original_put"),
test_reordered_attempts(content),
test_stale_pipeline_cannot_publish_newer_result(content),
test_committed_outbox_survives_publisher_crash(),
test_outbox_confirm_gap_republishes_same_job(),
test_image_processing_has_a_hard_timeout(),
]
report = {
"result": "PASS",
"track": "2",
"fixture": "tests/fixtures/sample.png",
"scenarios": results,
"invariants": {
"confirmed_work_eventually_ready": True,
"duplicate_effects_create_no_second_variant_set": True,
"stale_attempt_writes_no_effects": True,
"stale_pipeline_cannot_publish_current_metadata": True,
"ambiguous_confirm_republishes_stable_job": True,
"delivery_semantics": "at-least-once; effects are fenced/idempotent",
},
}
evidence = ROOT / "evidence" / "pipeline-faults.json"
evidence.parent.mkdir(exist_ok=True)
evidence.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps(report, sort_keys=True))
print(f"Evidence written to {evidence.relative_to(ROOT)}")
if __name__ == "__main__":
main()