forked from mtusi2026/mtusi2026lab5
68 lines
2.5 KiB
Python
68 lines
2.5 KiB
Python
"""Fault injection points for automated failure tests (track 5). Inert unless FAULT_INJECTION=1.
|
|
|
|
A fault is armed by the operator CLI (python -m app.ops fault arm ...) as a row in fault_injections and is
|
|
consumed atomically by exactly one hit. Actions:
|
|
crash os._exit(137): the process dies like SIGKILL, nothing after this point runs (no ack, no COMMIT)
|
|
delay:<s> sleep s seconds (widen a race window, e.g. let a newer pipeline version commit first)
|
|
transient raise a temporary failure -> retry with backoff
|
|
permanent raise a permanent failure -> failed without retries
|
|
|
|
Points: worker.after_claim, worker.before_transform, worker.after_s3_before_commit,
|
|
worker.after_commit_before_ack, relay.after_confirm_before_mark
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import sys
|
|
import uuid
|
|
from typing import Optional
|
|
|
|
from app import config
|
|
|
|
logger = logging.getLogger("faults")
|
|
|
|
POINTS = {
|
|
"worker.after_claim",
|
|
"worker.before_transform",
|
|
"worker.after_s3_before_commit",
|
|
"worker.after_commit_before_ack",
|
|
"relay.after_confirm_before_mark",
|
|
}
|
|
|
|
|
|
async def hit(pool, point: str, attachment_id: Optional[uuid.UUID], actor: str) -> None:
|
|
if not config.FAULT_INJECTION:
|
|
return
|
|
async with pool.acquire() as conn:
|
|
action = await conn.fetchval(
|
|
"""
|
|
UPDATE fault_injections SET remaining = remaining - 1
|
|
WHERE id = (SELECT id FROM fault_injections
|
|
WHERE point = $1 AND (attachment_id IS NULL OR attachment_id = $2) AND remaining > 0
|
|
ORDER BY id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
|
RETURNING action
|
|
""",
|
|
point, attachment_id,
|
|
)
|
|
if action is None:
|
|
return
|
|
await conn.execute(
|
|
"INSERT INTO job_events (attachment_id, event, actor, detail) VALUES ($1, 'fault_injected', $2, $3)",
|
|
attachment_id, actor, point + " -> " + action,
|
|
)
|
|
logger.warning("FAULT %s -> %s (attachment %s)", point, action, attachment_id)
|
|
if action == "crash":
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
os._exit(137)
|
|
if action.startswith("delay:"):
|
|
await asyncio.sleep(float(action.split(":", 1)[1]))
|
|
return
|
|
from app.imaging import ImageRejected, TransientFailure
|
|
if action == "transient":
|
|
raise TransientFailure("injected_transient")
|
|
if action == "permanent":
|
|
raise ImageRejected("processing_failed")
|