Files
mtusi2026lab5/app/faults.py

68 lines
2.5 KiB
Python

"""Fault injection points for automated failure tests (track 5). Inert unless FAULT_INJECTION=1.
A fault is armed by the operator CLI (python -m app.ops fault arm ...) as a row in fault_injections and is
consumed atomically by exactly one hit. Actions:
crash os._exit(137): the process dies like SIGKILL, nothing after this point runs (no ack, no COMMIT)
delay:<s> sleep s seconds (widen a race window, e.g. let a newer pipeline version commit first)
transient raise a temporary failure -> retry with backoff
permanent raise a permanent failure -> failed without retries
Points: worker.after_claim, worker.before_transform, worker.after_s3_before_commit,
worker.after_commit_before_ack, relay.after_confirm_before_mark
"""
from __future__ import annotations
import asyncio
import logging
import os
import sys
import uuid
from typing import Optional
from app import config
logger = logging.getLogger("faults")
POINTS = {
"worker.after_claim",
"worker.before_transform",
"worker.after_s3_before_commit",
"worker.after_commit_before_ack",
"relay.after_confirm_before_mark",
}
async def hit(pool, point: str, attachment_id: Optional[uuid.UUID], actor: str) -> None:
if not config.FAULT_INJECTION:
return
async with pool.acquire() as conn:
action = await conn.fetchval(
"""
UPDATE fault_injections SET remaining = remaining - 1
WHERE id = (SELECT id FROM fault_injections
WHERE point = $1 AND (attachment_id IS NULL OR attachment_id = $2) AND remaining > 0
ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED)
RETURNING action
""",
point, attachment_id,
)
if action is None:
return
await conn.execute(
"INSERT INTO job_events (attachment_id, event, actor, detail) VALUES ($1, 'fault_injected', $2, $3)",
attachment_id, actor, point + " -> " + action,
)
logger.warning("FAULT %s -> %s (attachment %s)", point, action, attachment_id)
if action == "crash":
sys.stdout.flush()
sys.stderr.flush()
os._exit(137)
if action.startswith("delay:"):
await asyncio.sleep(float(action.split(":", 1)[1]))
return
from app.imaging import ImageRejected, TransientFailure
if action == "transient":
raise TransientFailure("injected_transient")
if action == "permanent":
raise ImageRejected("processing_failed")