Files
mtusi2026lab6-fork/student-tests/test_pipeline_faults.py
T

311 lines
17 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""lab5, уровень 4: внедрённые отказы pipeline. Запуск — внутри compose-сети:
python scripts/ops.py pipeline-faults
(пересоздаёт worker с compose.faults.yaml, поднимает api-fault-publish/api-fault-dup,
запускает этот файл в one-off контейнере образа API, затем возвращает обычный worker).
Нужен прямой доступ к PostgreSQL (журнал job_events — доказательство того, ЧТО
произошло, а не только итогового статуса), к S3 и брокеру — берутся из тех же
secrets, что у API. Без окружения тесты пропускаются.
Сценарии и ожидаемые инварианты:
A. crash между COMMIT и publish (api-fault-publish) -> задачу публикует sweeper
другого процесса, ready без ручного создания нового задания;
B. crash после XADD до подтверждения в outbox (api-fault-dup) -> дубль сообщения,
worker делает no-op, ровно один succeeded и один набор вариантов;
C. worker умер после записи объектов в S3 до COMMIT -> после рестарта тот же
consumer забирает свой PEL, перезаписывает ТЕ ЖЕ байты, ready;
D. worker умер после COMMIT до XACK -> повторная доставка -> no-op already_succeeded;
E. временная ошибка -> retry с backoff+jitter (интервалы в заданных границах), ready на 3-й попытке;
F. постоянная ошибка -> failed сразу (без повторов), DLQ, content 409, прикрепить нельзя (409);
G. попытки исчерпаны -> failed retry_exhausted -> явный replay -> ready; повторный
replay успешного задания -> no-op (не исполняется дубль);
H. зависший worker (обработка дольше аренды) -> lease_expired, новая попытка у
другого consumer, ready; проснувшийся старый результат НЕ публикует (fencing, lease_lost);
I. staging удалён после безопасной фиксации;
J. мусорное сообщение в брокере (не по схеме) -> ack без падения worker.
"""
import hashlib
import os
import re
import sys
import time
import unittest
import uuid
from pathlib import Path
from urllib.parse import quote
HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(HERE))
sys.path.insert(0, str(HERE.parent))
from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402
FAULT_PUBLISH_URL = os.environ.get("FAULT_PUBLISH_URL", "").rstrip("/")
FAULT_DUP_URL = os.environ.get("FAULT_DUP_URL", "").rstrip("/")
FIX = HERE / "fixtures"
def _db_available():
try:
from src.config import load_settings
load_settings()
return True
except Exception:
return False
DB_OK = _db_available()
def upload(token, chat_id, data, filename, base_url=None, ctype="image/png"):
return request("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, quote(filename)),
token=token, raw=data, headers={"Content-Type": ctype}, base_url=base_url, timeout=60)
@unittest.skipUnless(reachable(BASE_URL) and DB_OK, "needs API at %s and DB/S3/broker env" % BASE_URL)
class PipelineFaultTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
from src.config import load_settings
from src.db import create_pool
from src.jobs import JobStore
from src.objectstore import ObjectStore
cls.settings = load_settings()
cls.pool = create_pool(cls.settings)
cls.pool.wait(timeout=15)
cls.jobs = JobStore(cls.pool, cls.settings)
cls.store = ObjectStore(cls.settings)
cls.alice = actor("flt_a")
cls.bob = actor("flt_b")
cls.chat = create_chat(cls.alice, cls.bob)
cls.png = (HERE.parent / "tests/fixtures/sample.png").read_bytes()
@classmethod
def tearDownClass(cls):
cls.pool.close()
# ------------------------------------------------------------ helpers
def accept(self, filename, base_url=None, data=None):
r = upload(self.alice["_token"], self.chat["id"], data or self.png, filename, base_url)
self.assertEqual(r.status, 202, (r.status, r.json))
self.assertEqual(r.json["status"], "queued")
return r.json
def job(self, att_id):
with self.pool.connection() as conn:
row = conn.execute("SELECT job_id, status, attempt, max_attempts, replay_count, last_error_code "
"FROM attachment_jobs WHERE attachment_id = %s", (att_id,)).fetchone()
return dict(zip(("job_id", "status", "attempt", "max_attempts", "replay_count", "last_error_code"), row))
def events(self, att_id):
with self.pool.connection() as conn:
return conn.execute(
"SELECT event, attempt, COALESCE(detail, ''), COALESCE(actor, ''), "
"EXTRACT(EPOCH FROM at)::float FROM job_events WHERE job_id IN "
"(SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", (att_id,)).fetchall()
def names(self, att_id):
return [e[0] for e in self.events(att_id)]
def wait_event(self, att_id, event, timeout=60.0, detail=None):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
for e in self.events(att_id):
if e[0] == event and (detail is None or detail in e[2]):
return e
time.sleep(0.3)
self.fail("event %s (%s) not seen for %s; events: %s" % (event, detail, att_id, self.names(att_id)))
def assert_single_result(self, att_id):
ready = request("GET", "/api/v1/attachments/" + att_id, token=self.bob["_token"]).json
self.assertEqual(ready["status"], "ready")
self.assertEqual(sorted(v["name"] for v in ready["variants"]), ["crop", "optimized", "thumbnail"])
with self.pool.connection() as conn:
n = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id = %s",
(att_id,)).fetchone()[0]
self.assertEqual(n, 3, "exactly one set of variants")
self.assertEqual(self.names(att_id).count("succeeded"), 1, "exactly one published result")
return ready
def reference_variants(self):
"""Эталон: те же байты, загруженные без отказов. Детерминизм вывода."""
if not hasattr(self.__class__, "_ref"):
att = self.accept("reference.png")
self.__class__._ref = {v["name"]: v["sha256"]
for v in wait_ready(self.bob["_token"], att["id"])["variants"]}
return self.__class__._ref
# ------------------------------------------------------------ A, B: outbox
@unittest.skipUnless(FAULT_PUBLISH_URL and reachable(FAULT_PUBLISH_URL), "api-fault-publish is not running")
def test_a_crash_between_commit_and_publish(self):
att = self.accept("a.png", base_url=FAULT_PUBLISH_URL) # 202 получен ДО падения
ready = wait_ready(self.bob["_token"], att["id"], timeout=60)
ev = self.events(att["id"])
fault = [e for e in ev if e[0] == "fault_injected"]
self.assertTrue(fault, "fault instance did not crash before publish")
published = [e for e in ev if e[0] == "published"]
self.assertEqual(len(published), 1)
self.assertNotEqual(published[0][3], fault[0][3], "row must be published by ANOTHER process")
self.assertFalse(reachable(FAULT_PUBLISH_URL), "fault instance should be dead")
self.assertEqual(self.job(att["id"])["attempt"], 1, "same job, no new job created manually")
self.assert_single_result(att["id"])
self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants())
@unittest.skipUnless(FAULT_DUP_URL and reachable(FAULT_DUP_URL), "api-fault-dup is not running")
def test_b_crash_after_publish_duplicate_is_noop(self):
att = self.accept("b.png", base_url=FAULT_DUP_URL)
wait_ready(self.bob["_token"], att["id"], timeout=60)
self.wait_event(att["id"], "redelivery_noop", timeout=30)
self.assert_single_result(att["id"])
# ------------------------------------------------------------ C, D: worker crash
def test_c_worker_crash_after_side_effects_before_commit(self):
att = self.accept("fault-crash-after-effects.png")
ready = wait_ready(self.bob["_token"], att["id"], timeout=120)
names = self.names(att["id"])
self.assertIn("fault_injected", names)
self.assertIn("lease_takeover", names, "restarted consumer must take over its own stale lease")
self.assert_single_result(att["id"])
self.assertEqual(self.job(att["id"])["attempt"], 1, "redelivery does not create a new attempt")
self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants(),
"repeated side effect writes identical bytes")
def test_d_worker_crash_after_commit_before_ack(self):
att = self.accept("fault-crash-before-ack.png")
wait_ready(self.bob["_token"], att["id"], timeout=60)
self.wait_event(att["id"], "fault_injected", timeout=30)
self.wait_event(att["id"], "redelivery_noop", timeout=120, detail="already_succeeded")
self.assert_single_result(att["id"])
# ------------------------------------------------------------ E, F, G: retry / DLQ / replay
def test_e_transient_errors_retry_with_backoff(self):
att = self.accept("fault-transient-until-3.png")
wait_ready(self.bob["_token"], att["id"], timeout=60)
ev = self.events(att["id"])
retries = [e for e in ev if e[0] == "retry_scheduled"]
self.assertEqual([e[1] for e in retries], [1, 2])
claims = [e for e in ev if e[0] == "claimed"]
self.assertEqual([e[1] for e in claims], [1, 2, 3])
# Значения worker'а из compose.faults.yaml (у one-off контейнера свои, обычные).
base, cap = self.settings_value("FAULTS_RETRY_BASE_SECONDS", 1.0), self.settings_value("FAULTS_RETRY_MAX_SECONDS", 4.0)
for (r, nxt) in zip(retries, claims[1:]):
delay = float(re.search(r"in ([0-9.]+)s", r[2]).group(1))
d = min(cap, base * 2 ** (r[1] - 1))
self.assertGreaterEqual(delay, d / 2 - 1e-6)
self.assertLessEqual(delay, d + 1e-6)
self.assertGreaterEqual(nxt[4] - r[4], delay - 0.05, "next attempt must not start before backoff")
self.assertEqual(self.job(att["id"])["attempt"], 3)
self.assert_single_result(att["id"])
def settings_value(self, name, default):
return float(os.environ.get(name) or default)
def test_f_permanent_error_fails_without_retry(self):
att = self.accept("fault-permanent.png")
done = wait_ready(self.bob["_token"], att["id"], timeout=60, allow_failed=True)
self.assertEqual((done["status"], done["error_code"], done["variants"]), ("failed", "invalid_image", []))
self.assertNotIn("retry_scheduled", self.names(att["id"]))
self.assertEqual(self.job(att["id"])["attempt"], 1)
with self.pool.connection() as conn:
dl = conn.execute("SELECT error_code FROM dead_letters WHERE attachment_id = %s", (att["id"],)).fetchone()
self.assertEqual(dl[0], "invalid_image")
r = request("GET", "/api/v1/attachments/%s/content" % att["id"], token=self.bob["_token"])
self.assertEqual((r.status, r.json["error"]["code"]), (409, "conflict"))
m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"],
payload={"text": "x", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
self.assertEqual(m.status, 409)
def test_g_retry_exhausted_dlq_then_explicit_replay(self):
att = self.accept("fault-transient-until-replay.png")
done = wait_ready(self.bob["_token"], att["id"], timeout=90, allow_failed=True)
self.assertEqual((done["status"], done["error_code"]), ("failed", "retry_exhausted"))
job = self.job(att["id"])
self.assertEqual((job["status"], job["attempt"]), ("failed", job["max_attempts"]))
self.assertEqual(self.names(att["id"]).count("retry_scheduled"), job["max_attempts"] - 1)
outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay")
self.assertEqual(outcome, "replayed")
wait_ready(self.bob["_token"], att["id"], timeout=60)
job2 = self.job(att["id"])
self.assertEqual((job2["status"], job2["attempt"], job2["replay_count"]),
("succeeded", job["attempt"] + 1, 1))
# Повторный replay успешного задания — no-op: дубль не исполняется.
outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay")
self.assertEqual(outcome, "noop")
self.assertEqual(self.job(att["id"])["attempt"], job2["attempt"])
with self.pool.connection() as conn:
dl = conn.execute("SELECT replayed_at IS NOT NULL, replay_count FROM dead_letters WHERE attachment_id = %s",
(att["id"],)).fetchone()
self.assertEqual(dl, (True, 1))
self.assert_single_result(att["id"])
# ------------------------------------------------------------ H: hung worker
def test_h_hung_worker_lease_expires_and_is_fenced(self):
att = self.accept("fault-slow.png")
wait_ready(self.bob["_token"], att["id"], timeout=90)
self.wait_event(att["id"], "lease_expired", timeout=5)
self.wait_event(att["id"], "lease_lost", timeout=80)
job = self.job(att["id"])
self.assertEqual(job["attempt"], 2)
self.assert_single_result(att["id"])
# ------------------------------------------------------------ I, J
def test_i_staging_removed_after_safe_commit(self):
att = self.accept("plain.png")
wait_ready(self.bob["_token"], att["id"], timeout=60)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
with self.pool.connection() as conn:
key = conn.execute("SELECT staging_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0]
if key is None:
break
time.sleep(0.3)
self.assertIsNone(key)
with self.pool.connection() as conn:
final = conn.execute("SELECT object_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0]
self.assertTrue(final.startswith("final/"))
data = self.store.read_all(final, 20 * 1024 * 1024)
self.assertEqual(hashlib.sha256(data).hexdigest(), att["sha256"])
with self.pool.connection() as conn:
leftovers = conn.execute(
"SELECT count(*) FROM job_events WHERE event = 'accepted' AND attachment_id = %s", (att["id"],)
).fetchone()[0]
self.assertEqual(leftovers, 1)
def test_j_garbage_broker_message_is_acked(self):
from src.broker import Broker
b = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group)
mid = b.client.xadd(self.settings.broker_stream, {"job": '{"not":"a job"}'}).decode()
mid2 = b.publish({"schema_version": 1, "job_id": str(uuid.uuid4()), "type": "attachment.process.v1",
"attachment_id": str(uuid.uuid4()), "pipeline_version": 1, "attempt": 1,
"created_at": "2026-01-01T00:00:00Z"})
deadline = time.monotonic() + 20
pending = None
while time.monotonic() < deadline:
pending = [p for p in b.client.xpending_range(self.settings.broker_stream, self.settings.broker_group,
"-", "+", 100)
if p["message_id"].decode() in (mid, mid2)]
info = b.client.xinfo_groups(self.settings.broker_stream)[0]
delivered = info["last-delivered-id"].decode()
if not pending and _id(delivered) >= _id(mid2):
break
time.sleep(0.3)
self.assertEqual(pending, [], "poison/unknown messages must be acknowledged")
b.close()
def _id(x):
a, _, b = x.partition("-")
return int(a), int(b or 0)
if __name__ == "__main__":
unittest.main()