forked from mtusi2026/mtusi2026lab6
311 lines
17 KiB
Python
311 lines
17 KiB
Python
"""lab5, уровень 4: внедрённые отказы pipeline. Запуск — внутри compose-сети:
|
||
|
||
python scripts/ops.py pipeline-faults
|
||
|
||
(пересоздаёт worker с compose.faults.yaml, поднимает api-fault-publish/api-fault-dup,
|
||
запускает этот файл в one-off контейнере образа API, затем возвращает обычный worker).
|
||
|
||
Нужен прямой доступ к PostgreSQL (журнал job_events — доказательство того, ЧТО
|
||
произошло, а не только итогового статуса), к S3 и брокеру — берутся из тех же
|
||
secrets, что у API. Без окружения тесты пропускаются.
|
||
|
||
Сценарии и ожидаемые инварианты:
|
||
A. crash между COMMIT и publish (api-fault-publish) -> задачу публикует sweeper
|
||
другого процесса, ready без ручного создания нового задания;
|
||
B. crash после XADD до подтверждения в outbox (api-fault-dup) -> дубль сообщения,
|
||
worker делает no-op, ровно один succeeded и один набор вариантов;
|
||
C. worker умер после записи объектов в S3 до COMMIT -> после рестарта тот же
|
||
consumer забирает свой PEL, перезаписывает ТЕ ЖЕ байты, ready;
|
||
D. worker умер после COMMIT до XACK -> повторная доставка -> no-op already_succeeded;
|
||
E. временная ошибка -> retry с backoff+jitter (интервалы в заданных границах), ready на 3-й попытке;
|
||
F. постоянная ошибка -> failed сразу (без повторов), DLQ, content 409, прикрепить нельзя (409);
|
||
G. попытки исчерпаны -> failed retry_exhausted -> явный replay -> ready; повторный
|
||
replay успешного задания -> no-op (не исполняется дубль);
|
||
H. зависший worker (обработка дольше аренды) -> lease_expired, новая попытка у
|
||
другого consumer, ready; проснувшийся старый результат НЕ публикует (fencing, lease_lost);
|
||
I. staging удалён после безопасной фиксации;
|
||
J. мусорное сообщение в брокере (не по схеме) -> ack без падения worker.
|
||
"""
|
||
import hashlib
|
||
import os
|
||
import re
|
||
import sys
|
||
import time
|
||
import unittest
|
||
import uuid
|
||
from pathlib import Path
|
||
from urllib.parse import quote
|
||
|
||
HERE = Path(__file__).resolve().parent
|
||
sys.path.insert(0, str(HERE))
|
||
sys.path.insert(0, str(HERE.parent))
|
||
from common import BASE_URL, actor, create_chat, reachable, request, wait_ready # noqa: E402
|
||
|
||
FAULT_PUBLISH_URL = os.environ.get("FAULT_PUBLISH_URL", "").rstrip("/")
|
||
FAULT_DUP_URL = os.environ.get("FAULT_DUP_URL", "").rstrip("/")
|
||
FIX = HERE / "fixtures"
|
||
|
||
|
||
def _db_available():
|
||
try:
|
||
from src.config import load_settings
|
||
load_settings()
|
||
return True
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
DB_OK = _db_available()
|
||
|
||
|
||
def upload(token, chat_id, data, filename, base_url=None, ctype="image/png"):
|
||
return request("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, quote(filename)),
|
||
token=token, raw=data, headers={"Content-Type": ctype}, base_url=base_url, timeout=60)
|
||
|
||
|
||
@unittest.skipUnless(reachable(BASE_URL) and DB_OK, "needs API at %s and DB/S3/broker env" % BASE_URL)
|
||
class PipelineFaultTests(unittest.TestCase):
|
||
|
||
@classmethod
|
||
def setUpClass(cls):
|
||
from src.config import load_settings
|
||
from src.db import create_pool
|
||
from src.jobs import JobStore
|
||
from src.objectstore import ObjectStore
|
||
cls.settings = load_settings()
|
||
cls.pool = create_pool(cls.settings)
|
||
cls.pool.wait(timeout=15)
|
||
cls.jobs = JobStore(cls.pool, cls.settings)
|
||
cls.store = ObjectStore(cls.settings)
|
||
cls.alice = actor("flt_a")
|
||
cls.bob = actor("flt_b")
|
||
cls.chat = create_chat(cls.alice, cls.bob)
|
||
cls.png = (HERE.parent / "tests/fixtures/sample.png").read_bytes()
|
||
|
||
@classmethod
|
||
def tearDownClass(cls):
|
||
cls.pool.close()
|
||
|
||
# ------------------------------------------------------------ helpers
|
||
|
||
def accept(self, filename, base_url=None, data=None):
|
||
r = upload(self.alice["_token"], self.chat["id"], data or self.png, filename, base_url)
|
||
self.assertEqual(r.status, 202, (r.status, r.json))
|
||
self.assertEqual(r.json["status"], "queued")
|
||
return r.json
|
||
|
||
def job(self, att_id):
|
||
with self.pool.connection() as conn:
|
||
row = conn.execute("SELECT job_id, status, attempt, max_attempts, replay_count, last_error_code "
|
||
"FROM attachment_jobs WHERE attachment_id = %s", (att_id,)).fetchone()
|
||
return dict(zip(("job_id", "status", "attempt", "max_attempts", "replay_count", "last_error_code"), row))
|
||
|
||
def events(self, att_id):
|
||
with self.pool.connection() as conn:
|
||
return conn.execute(
|
||
"SELECT event, attempt, COALESCE(detail, ''), COALESCE(actor, ''), "
|
||
"EXTRACT(EPOCH FROM at)::float FROM job_events WHERE job_id IN "
|
||
"(SELECT job_id FROM attachment_jobs WHERE attachment_id = %s) ORDER BY id", (att_id,)).fetchall()
|
||
|
||
def names(self, att_id):
|
||
return [e[0] for e in self.events(att_id)]
|
||
|
||
def wait_event(self, att_id, event, timeout=60.0, detail=None):
|
||
deadline = time.monotonic() + timeout
|
||
while time.monotonic() < deadline:
|
||
for e in self.events(att_id):
|
||
if e[0] == event and (detail is None or detail in e[2]):
|
||
return e
|
||
time.sleep(0.3)
|
||
self.fail("event %s (%s) not seen for %s; events: %s" % (event, detail, att_id, self.names(att_id)))
|
||
|
||
def assert_single_result(self, att_id):
|
||
ready = request("GET", "/api/v1/attachments/" + att_id, token=self.bob["_token"]).json
|
||
self.assertEqual(ready["status"], "ready")
|
||
self.assertEqual(sorted(v["name"] for v in ready["variants"]), ["crop", "optimized", "thumbnail"])
|
||
with self.pool.connection() as conn:
|
||
n = conn.execute("SELECT count(*) FROM attachment_variants WHERE attachment_id = %s",
|
||
(att_id,)).fetchone()[0]
|
||
self.assertEqual(n, 3, "exactly one set of variants")
|
||
self.assertEqual(self.names(att_id).count("succeeded"), 1, "exactly one published result")
|
||
return ready
|
||
|
||
def reference_variants(self):
|
||
"""Эталон: те же байты, загруженные без отказов. Детерминизм вывода."""
|
||
if not hasattr(self.__class__, "_ref"):
|
||
att = self.accept("reference.png")
|
||
self.__class__._ref = {v["name"]: v["sha256"]
|
||
for v in wait_ready(self.bob["_token"], att["id"])["variants"]}
|
||
return self.__class__._ref
|
||
|
||
# ------------------------------------------------------------ A, B: outbox
|
||
|
||
@unittest.skipUnless(FAULT_PUBLISH_URL and reachable(FAULT_PUBLISH_URL), "api-fault-publish is not running")
|
||
def test_a_crash_between_commit_and_publish(self):
|
||
att = self.accept("a.png", base_url=FAULT_PUBLISH_URL) # 202 получен ДО падения
|
||
ready = wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
ev = self.events(att["id"])
|
||
fault = [e for e in ev if e[0] == "fault_injected"]
|
||
self.assertTrue(fault, "fault instance did not crash before publish")
|
||
published = [e for e in ev if e[0] == "published"]
|
||
self.assertEqual(len(published), 1)
|
||
self.assertNotEqual(published[0][3], fault[0][3], "row must be published by ANOTHER process")
|
||
self.assertFalse(reachable(FAULT_PUBLISH_URL), "fault instance should be dead")
|
||
self.assertEqual(self.job(att["id"])["attempt"], 1, "same job, no new job created manually")
|
||
self.assert_single_result(att["id"])
|
||
self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants())
|
||
|
||
@unittest.skipUnless(FAULT_DUP_URL and reachable(FAULT_DUP_URL), "api-fault-dup is not running")
|
||
def test_b_crash_after_publish_duplicate_is_noop(self):
|
||
att = self.accept("b.png", base_url=FAULT_DUP_URL)
|
||
wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
self.wait_event(att["id"], "redelivery_noop", timeout=30)
|
||
self.assert_single_result(att["id"])
|
||
|
||
# ------------------------------------------------------------ C, D: worker crash
|
||
|
||
def test_c_worker_crash_after_side_effects_before_commit(self):
|
||
att = self.accept("fault-crash-after-effects.png")
|
||
ready = wait_ready(self.bob["_token"], att["id"], timeout=120)
|
||
names = self.names(att["id"])
|
||
self.assertIn("fault_injected", names)
|
||
self.assertIn("lease_takeover", names, "restarted consumer must take over its own stale lease")
|
||
self.assert_single_result(att["id"])
|
||
self.assertEqual(self.job(att["id"])["attempt"], 1, "redelivery does not create a new attempt")
|
||
self.assertEqual({v["name"]: v["sha256"] for v in ready["variants"]}, self.reference_variants(),
|
||
"repeated side effect writes identical bytes")
|
||
|
||
def test_d_worker_crash_after_commit_before_ack(self):
|
||
att = self.accept("fault-crash-before-ack.png")
|
||
wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
self.wait_event(att["id"], "fault_injected", timeout=30)
|
||
self.wait_event(att["id"], "redelivery_noop", timeout=120, detail="already_succeeded")
|
||
self.assert_single_result(att["id"])
|
||
|
||
# ------------------------------------------------------------ E, F, G: retry / DLQ / replay
|
||
|
||
def test_e_transient_errors_retry_with_backoff(self):
|
||
att = self.accept("fault-transient-until-3.png")
|
||
wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
ev = self.events(att["id"])
|
||
retries = [e for e in ev if e[0] == "retry_scheduled"]
|
||
self.assertEqual([e[1] for e in retries], [1, 2])
|
||
claims = [e for e in ev if e[0] == "claimed"]
|
||
self.assertEqual([e[1] for e in claims], [1, 2, 3])
|
||
# Значения worker'а из compose.faults.yaml (у one-off контейнера свои, обычные).
|
||
base, cap = self.settings_value("FAULTS_RETRY_BASE_SECONDS", 1.0), self.settings_value("FAULTS_RETRY_MAX_SECONDS", 4.0)
|
||
for (r, nxt) in zip(retries, claims[1:]):
|
||
delay = float(re.search(r"in ([0-9.]+)s", r[2]).group(1))
|
||
d = min(cap, base * 2 ** (r[1] - 1))
|
||
self.assertGreaterEqual(delay, d / 2 - 1e-6)
|
||
self.assertLessEqual(delay, d + 1e-6)
|
||
self.assertGreaterEqual(nxt[4] - r[4], delay - 0.05, "next attempt must not start before backoff")
|
||
self.assertEqual(self.job(att["id"])["attempt"], 3)
|
||
self.assert_single_result(att["id"])
|
||
|
||
def settings_value(self, name, default):
|
||
return float(os.environ.get(name) or default)
|
||
|
||
def test_f_permanent_error_fails_without_retry(self):
|
||
att = self.accept("fault-permanent.png")
|
||
done = wait_ready(self.bob["_token"], att["id"], timeout=60, allow_failed=True)
|
||
self.assertEqual((done["status"], done["error_code"], done["variants"]), ("failed", "invalid_image", []))
|
||
self.assertNotIn("retry_scheduled", self.names(att["id"]))
|
||
self.assertEqual(self.job(att["id"])["attempt"], 1)
|
||
with self.pool.connection() as conn:
|
||
dl = conn.execute("SELECT error_code FROM dead_letters WHERE attachment_id = %s", (att["id"],)).fetchone()
|
||
self.assertEqual(dl[0], "invalid_image")
|
||
r = request("GET", "/api/v1/attachments/%s/content" % att["id"], token=self.bob["_token"])
|
||
self.assertEqual((r.status, r.json["error"]["code"]), (409, "conflict"))
|
||
m = request("POST", "/api/v1/chats/%s/messages" % self.chat["id"], token=self.alice["_token"],
|
||
payload={"text": "x", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||
self.assertEqual(m.status, 409)
|
||
|
||
def test_g_retry_exhausted_dlq_then_explicit_replay(self):
|
||
att = self.accept("fault-transient-until-replay.png")
|
||
done = wait_ready(self.bob["_token"], att["id"], timeout=90, allow_failed=True)
|
||
self.assertEqual((done["status"], done["error_code"]), ("failed", "retry_exhausted"))
|
||
job = self.job(att["id"])
|
||
self.assertEqual((job["status"], job["attempt"]), ("failed", job["max_attempts"]))
|
||
self.assertEqual(self.names(att["id"]).count("retry_scheduled"), job["max_attempts"] - 1)
|
||
outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay")
|
||
self.assertEqual(outcome, "replayed")
|
||
wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
job2 = self.job(att["id"])
|
||
self.assertEqual((job2["status"], job2["attempt"], job2["replay_count"]),
|
||
("succeeded", job["attempt"] + 1, 1))
|
||
# Повторный replay успешного задания — no-op: дубль не исполняется.
|
||
outcome, _ = self.jobs.replay(uuid.UUID(att["id"]), actor="test:replay")
|
||
self.assertEqual(outcome, "noop")
|
||
self.assertEqual(self.job(att["id"])["attempt"], job2["attempt"])
|
||
with self.pool.connection() as conn:
|
||
dl = conn.execute("SELECT replayed_at IS NOT NULL, replay_count FROM dead_letters WHERE attachment_id = %s",
|
||
(att["id"],)).fetchone()
|
||
self.assertEqual(dl, (True, 1))
|
||
self.assert_single_result(att["id"])
|
||
|
||
# ------------------------------------------------------------ H: hung worker
|
||
|
||
def test_h_hung_worker_lease_expires_and_is_fenced(self):
|
||
att = self.accept("fault-slow.png")
|
||
wait_ready(self.bob["_token"], att["id"], timeout=90)
|
||
self.wait_event(att["id"], "lease_expired", timeout=5)
|
||
self.wait_event(att["id"], "lease_lost", timeout=80)
|
||
job = self.job(att["id"])
|
||
self.assertEqual(job["attempt"], 2)
|
||
self.assert_single_result(att["id"])
|
||
|
||
# ------------------------------------------------------------ I, J
|
||
|
||
def test_i_staging_removed_after_safe_commit(self):
|
||
att = self.accept("plain.png")
|
||
wait_ready(self.bob["_token"], att["id"], timeout=60)
|
||
deadline = time.monotonic() + 15
|
||
while time.monotonic() < deadline:
|
||
with self.pool.connection() as conn:
|
||
key = conn.execute("SELECT staging_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0]
|
||
if key is None:
|
||
break
|
||
time.sleep(0.3)
|
||
self.assertIsNone(key)
|
||
with self.pool.connection() as conn:
|
||
final = conn.execute("SELECT object_key FROM attachments WHERE id = %s", (att["id"],)).fetchone()[0]
|
||
self.assertTrue(final.startswith("final/"))
|
||
data = self.store.read_all(final, 20 * 1024 * 1024)
|
||
self.assertEqual(hashlib.sha256(data).hexdigest(), att["sha256"])
|
||
with self.pool.connection() as conn:
|
||
leftovers = conn.execute(
|
||
"SELECT count(*) FROM job_events WHERE event = 'accepted' AND attachment_id = %s", (att["id"],)
|
||
).fetchone()[0]
|
||
self.assertEqual(leftovers, 1)
|
||
|
||
def test_j_garbage_broker_message_is_acked(self):
|
||
from src.broker import Broker
|
||
b = Broker(self.settings.broker_url, self.settings.broker_stream, self.settings.broker_group)
|
||
mid = b.client.xadd(self.settings.broker_stream, {"job": '{"not":"a job"}'}).decode()
|
||
mid2 = b.publish({"schema_version": 1, "job_id": str(uuid.uuid4()), "type": "attachment.process.v1",
|
||
"attachment_id": str(uuid.uuid4()), "pipeline_version": 1, "attempt": 1,
|
||
"created_at": "2026-01-01T00:00:00Z"})
|
||
deadline = time.monotonic() + 20
|
||
pending = None
|
||
while time.monotonic() < deadline:
|
||
pending = [p for p in b.client.xpending_range(self.settings.broker_stream, self.settings.broker_group,
|
||
"-", "+", 100)
|
||
if p["message_id"].decode() in (mid, mid2)]
|
||
info = b.client.xinfo_groups(self.settings.broker_stream)[0]
|
||
delivered = info["last-delivered-id"].decode()
|
||
if not pending and _id(delivered) >= _id(mid2):
|
||
break
|
||
time.sleep(0.3)
|
||
self.assertEqual(pending, [], "poison/unknown messages must be acknowledged")
|
||
b.close()
|
||
|
||
|
||
def _id(x):
|
||
a, _, b = x.partition("-")
|
||
return int(a), int(b or 0)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|