forked from mtusi2026/mtusi2026lab7
355 lines
24 KiB
Python
355 lines
24 KiB
Python
#!/usr/bin/env python3
|
||
"""Уровень 4 и трек 2: автоматический fault-injection pipeline вложений и проверка инвариантов.
|
||
|
||
На время теста api и worker пересоздаются с FAILPOINTS=1 и JOB_TIMEOUT_SECONDS=10 (аренда 25 с), в конце —
|
||
снова с конфигурацией из .env. Отказы настоящие: аварийный выход процесса в точке (src/failpoints.ts), пауза процесса,
|
||
дубли и перестановка сообщений через outbox, остановка S3 и брокера, устаревшая job после новой версии pipeline.
|
||
|
||
Инварианты (проверяются в конце по всем вложениям тестового чата):
|
||
I1 нет потерянной работы: каждое вложение, для которого записано задание, в конечном состоянии ready/failed;
|
||
I2 один результат: у пары (вложение, версия) ровно 3 варианта, лишние объекты попыток убирает reconciler;
|
||
I3 новый результат не перезаписан старым: версия только растёт, варианты v2 не меняются после старых job;
|
||
I4 повтор даёт тот же результат: sha256 вариантов после сбоев = sha256 контрольной обработки тех же байтов;
|
||
I5 API согласован: каждый вариант и оригинал скачиваются, размер и sha256 совпадают с метаданными;
|
||
I6 все job в конечном состоянии (done/failed/superseded), reconciler после очистки ничего не находит.
|
||
Запуск: python3 student-tests/pipeline_faults.py --base-url http://localhost:8080"""
|
||
import hashlib
|
||
import http.client
|
||
import json
|
||
import os
|
||
import struct
|
||
import subprocess
|
||
import time
|
||
import urllib.parse
|
||
import uuid
|
||
import zlib
|
||
|
||
from failures import PNG, ROOT, sql, wait_ready
|
||
from client import Client, arguments, check, run
|
||
|
||
TEST_ENV = {'FAILPOINTS': '1', 'JOB_TIMEOUT_SECONDS': '10'} # аренда = таймаут + 15 с
|
||
LEASE = 25
|
||
|
||
|
||
def compose(*args, env=None):
|
||
r = subprocess.run(['docker', 'compose', *args], cwd=ROOT, text=True, capture_output=True, env={**os.environ, **(env or {})})
|
||
check(r.returncode == 0, 'docker compose %s: %s' % (args[0], r.stderr.strip().splitlines()[-1:]))
|
||
return r.stdout
|
||
|
||
|
||
def jobs(*args):
|
||
out = compose('run', '--rm', '--no-deps', '-T', 'migrate', 'node', 'src/main.ts', 'jobs', *args)
|
||
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
|
||
|
||
|
||
def reconcile(*flags):
|
||
"""Находки reconciler (по одной JSON-строке) и итоговая сводка."""
|
||
out = compose('run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'src/main.ts', 'reconcile', *flags)
|
||
lines = [json.loads(line) for line in out.splitlines() if line.startswith('{')]
|
||
return [x for x in lines if 'finding' in x], lines[-1]
|
||
|
||
|
||
def rows(query):
|
||
out = sql(query)
|
||
return [line.split('|') for line in out.splitlines()] if out else []
|
||
|
||
|
||
def wait_for(what, fn, seconds):
|
||
deadline = time.monotonic() + seconds
|
||
while True:
|
||
value = fn()
|
||
if value:
|
||
return value
|
||
check(time.monotonic() < deadline, 'timeout: ' + what)
|
||
time.sleep(1)
|
||
|
||
|
||
def healthy(service):
|
||
"""Все реплики сервиса healthy (после crash Docker перезапускает контейнер)."""
|
||
out = compose('ps', '-a', service, '--format', '{{.State}} {{.Health}}').split('\n')
|
||
return all(line == 'running healthy' for line in out if line)
|
||
|
||
|
||
def bad_filter_png():
|
||
"""Структура PNG верна (CRC, объём данных), но байт фильтра строк 9 не существует: API принимает, libvips — нет."""
|
||
def chunk(kind, data):
|
||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||
raw = b''.join(b'\x09' + bytes(64 * 3) for _ in range(64))
|
||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 64, 64, 8, 2, 0, 0, 0))
|
||
+ chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b''))
|
||
|
||
|
||
def main():
|
||
args = arguments('Pipeline fault injection with invariant checks')
|
||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||
since = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||
compose('up', '-d', '--no-deps', 'api', 'worker', env=TEST_ENV)
|
||
wait_for('api healthy with FAILPOINTS=1', lambda: healthy('api'), 90)
|
||
for service in ('api', 'worker'):
|
||
ids = compose('ps', '-q', service).split()
|
||
check(len(ids) == 2 and all(subprocess.run(['docker', 'exec', i, 'printenv', 'FAILPOINTS'], capture_output=True, text=True)
|
||
.stdout.strip() == '1' for i in ids), '%s replicas without FAILPOINTS=1' % service)
|
||
wait_ready(c)
|
||
alice, bob, _, chat = c.bootstrap()
|
||
report = []
|
||
|
||
def record(name, text):
|
||
report.append('%-44s %s' % (name, text))
|
||
print(report[-1], flush=True)
|
||
|
||
def upload(name, data=PNG):
|
||
path = '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], urllib.parse.quote(name))
|
||
return c.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']['id']
|
||
|
||
def raw_upload(name):
|
||
"""Upload, при котором API может погибнуть посреди запроса: статус без проверок клиента (0 — обрыв)."""
|
||
url = urllib.parse.urlsplit(c.base)
|
||
conn = http.client.HTTPConnection(url.hostname, url.port, timeout=30)
|
||
try:
|
||
conn.request('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), body=PNG,
|
||
headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'})
|
||
return conn.getresponse().status
|
||
except OSError:
|
||
return 0
|
||
finally:
|
||
conn.close()
|
||
|
||
def find(name):
|
||
return sql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name))
|
||
|
||
def state(aid):
|
||
return rows("SELECT state, coalesce(pipeline_version, 0), coalesce(error_code, '') FROM attachments WHERE id = '%s'" % aid)[0]
|
||
|
||
def job(aid, version=1):
|
||
r = rows("SELECT job_id, state, attempts, coalesce(error_code, '') FROM jobs WHERE attachment_id = '%s' AND pipeline_version = %d"
|
||
% (aid, version))
|
||
return r[0] if r else None
|
||
|
||
def variants(aid, version):
|
||
return {name: sha for name, sha in rows("SELECT name, sha256 FROM attachment_variants WHERE attachment_id = '%s' "
|
||
"AND pipeline_version = %d" % (aid, version))}
|
||
|
||
def settle(aid, seconds=90, version=1):
|
||
"""Дождаться конечного состояния вложения и его job нужной версии."""
|
||
return wait_for('attachment %s final' % aid, lambda: state(aid)[0] in ('ready', 'failed')
|
||
and (job(aid, version) or ['', ''])[1] in ('done', 'failed', 'superseded') and state(aid), seconds)
|
||
|
||
def deliveries(job_id):
|
||
log = compose('logs', '--no-log-prefix', '--since', since, 'worker')
|
||
return [line for line in log.splitlines() if job_id in line]
|
||
|
||
try:
|
||
# S0. Контроль: эталонные sha256 вариантов v1 для тех же байтов.
|
||
control = upload('control.png')
|
||
settle(control)
|
||
ref = variants(control, 1)
|
||
check(len(ref) == 3, 'control variants')
|
||
record('S0 контроль', 'ready v1, варианты %s' % ', '.join(sorted(ref)))
|
||
|
||
# S1. API падает после записи staging, до транзакции задания: 202 не было — работы не обещано.
|
||
status = raw_upload('failpoint-api_after_staging.png')
|
||
s1 = find('failpoint-api_after_staging.png')
|
||
check(status != 202 and state(s1)[0] == 'pending' and job(s1) is None, 'S1: %s %s' % (status, state(s1)))
|
||
wait_for('api restarted', lambda: healthy('api'), 90)
|
||
record('S1 API: crash после staging', 'клиент получил %d; строка pending, заданий нет → stale_pending для reconciler' % status)
|
||
|
||
# S2. API падает после COMMIT задания+outbox, до publish и ответа: публикует relay другой реплики.
|
||
status = raw_upload('failpoint-api_after_commit.png')
|
||
s2 = find('failpoint-api_after_commit.png')
|
||
st = settle(s2)
|
||
j = job(s2)
|
||
check(status != 202 and st[0] == 'ready' and j[1:3] == ['done', '1'], 'S2: %s %s %s' % (status, st, j))
|
||
wait_for('api restarted', lambda: healthy('api'), 90)
|
||
record('S2 API: crash после COMMIT, до publish', 'клиент %d; задание из outbox опубликовано → ready, попыток 1' % status)
|
||
|
||
# S3. Relay падает после publisher confirm, до отметки published_at: строку публикует снова — дубль.
|
||
status = raw_upload('failpoint-relay_after_publish.png')
|
||
s3 = find('failpoint-relay_after_publish.png')
|
||
st = settle(s3)
|
||
j = job(s3)
|
||
seen = wait_for('duplicate delivered', lambda: len(deliveries(j[0])) >= 2 and deliveries(j[0]), 20)
|
||
check(st[0] == 'ready' and j[1:3] == ['done', '1'] and len(seen) >= 2 and len(variants(s3, 1)) == 3, 'S3: %s %s %s' % (st, j, seen))
|
||
wait_for('api restarted', lambda: healthy('api'), 90)
|
||
record('S3 relay: crash после confirm', 'клиент %d; доставок worker: %d (дубль), обработка 1, вариантов 3' % (status, len(seen)))
|
||
|
||
# S4. Тройная доставка одной попытки одновременно двум worker.
|
||
compose('stop', 'worker')
|
||
s4 = upload('duplicate.png')
|
||
j4 = job(s4)[0]
|
||
sql("INSERT INTO outbox (job_id, attempt) VALUES ('%s', 1), ('%s', 1)" % (j4, j4))
|
||
wait_for('duplicates published', lambda: sql("SELECT count(*) FROM outbox WHERE job_id = '%s' AND published_at IS NOT NULL" % j4) == '3', 20)
|
||
compose('start', 'worker')
|
||
st = settle(s4)
|
||
seen = wait_for('all copies delivered', lambda: len(deliveries(j4)) >= 3 and deliveries(j4), 20)
|
||
noop = [line for line in seen if 'no-op' in line]
|
||
check(st[0] == 'ready' and job(s4)[2] == '1' and len(seen) == 3 and len(noop) == 2 and variants(s4, 1) == ref, 'S4: %s' % seen)
|
||
record('S4 дубль: 3 копии попытки 1', '3 доставки → 1 обработка, 2 no-op; попыток 1, варианты = контрольным')
|
||
|
||
# S5. Reorder в новой версии: сообщение попытки 2 опубликовано раньше попытки 1 (job v2 для ready v1).
|
||
j5 = str(uuid.uuid4())
|
||
sql("INSERT INTO jobs (job_id, attachment_id, pipeline_version) VALUES ('%s', '%s', 2); "
|
||
"INSERT INTO outbox (job_id, attempt) VALUES ('%s', 2), ('%s', 1)" % (j5, s4, j5, j5))
|
||
st = settle(s4, version=2)
|
||
v2 = variants(s4, 2)
|
||
seen = wait_for('attempt 2 no-op', lambda: any('попытка 2: no-op' in line for line in deliveries(j5)) and deliveries(j5), 20)
|
||
check(st[:2] == ['ready', '2'] and job(s4, 2)[1:3] == ['done', '1'] and len(v2) == 3 and v2['thumbnail'] != ref['thumbnail']
|
||
and any('попытка 2: no-op' in line for line in seen), 'S5: %s %s' % (st, seen))
|
||
record('S5 reorder: попытка 2 раньше 1 (job v2)', 'попытка 2 → no-op, попытка 1 → done; версия 1 → 2, вариантов v2: 3')
|
||
|
||
# S6. Устаревшая job v1 после публикации v2: поздний дубль старого сообщения и повторно поставленная v1.
|
||
j1 = job(s4)[0]
|
||
before = len(deliveries(j1))
|
||
sql("INSERT INTO outbox (job_id, attempt) VALUES ('%s', 1)" % j1)
|
||
wait_for('late duplicate handled', lambda: len(deliveries(j1)) > before, 20)
|
||
sql("UPDATE jobs SET state = 'queued' WHERE job_id = '%s'; "
|
||
"INSERT INTO outbox (job_id, attempt) SELECT job_id, attempts + 1 FROM jobs WHERE job_id = '%s'" % (j1, j1))
|
||
wait_for('stale v1 superseded', lambda: job(s4)[1] == 'superseded', 30)
|
||
check(state(s4)[:2] == ['ready', '2'] and variants(s4, 2) == v2, 'S6: v2 changed')
|
||
record('S6 устаревшая v1 после v2 (захват)', 'дубль старого сообщения → no-op; повторно поставленная v1 → superseded; v2 не тронута')
|
||
|
||
# S7. v1 фиксируется после v2: v1 «зависла» 10 с после записи объектов, тем временем v2 опубликована.
|
||
name = 'failpoint-worker_after_side_effect-10s.png'
|
||
s7 = upload(name)
|
||
wait_for('v1 stalled after side effects', lambda: sql("SELECT count(*) FROM failpoint_hits WHERE attachment_id = '%s'" % s7) == '1', 30)
|
||
j7 = str(uuid.uuid4())
|
||
sql("INSERT INTO jobs (job_id, attachment_id, pipeline_version) VALUES ('%s', '%s', 2); "
|
||
"INSERT INTO outbox (job_id, attempt) VALUES ('%s', 1)" % (j7, s7, j7))
|
||
wait_for('v2 done', lambda: (job(s7, 2) or ['', ''])[1] == 'done', 30)
|
||
v2_7 = variants(s7, 2)
|
||
wait_for('v1 superseded at commit', lambda: job(s7)[1] == 'superseded', 30)
|
||
check(state(s7)[:2] == ['ready', '2'] and variants(s7, 1) == {} and variants(s7, 2) == v2_7, 'S7: %s' % state(s7))
|
||
record('S7 v1 фиксируется после v2 (фиксация)', 'v2 ready; v1 при фиксации → superseded, вариантов v1 в БД 0, объекты удалены')
|
||
|
||
# S8–S10. Worker crash до side effect, после side effect, после COMMIT (до ack).
|
||
crashed = {}
|
||
for label, point, attempts in (('S8 worker: crash после захвата', 'worker_after_claim', '2'),
|
||
('S9 worker: crash после записи S3', 'worker_after_side_effect', '2'),
|
||
('S10 worker: crash после COMMIT до ack', 'worker_after_commit', '1')):
|
||
started = time.monotonic()
|
||
aid = crashed[point] = upload('failpoint-%s.png' % point)
|
||
st = settle(aid, LEASE * 3)
|
||
j = job(aid)
|
||
check(st[0] == 'ready' and j[1:3] == ['done', attempts] and variants(aid, 1) == ref, '%s: %s %s' % (point, st, j))
|
||
wait_for('worker restarted', lambda: set(compose('ps', '-a', 'worker', '--format', '{{.State}}').split()) == {'running'}, 30)
|
||
how = 'redelivery → no-op (аренда жива), аренда истекла → попытка 2' if attempts == '2' else 'redelivery → no-op (job done)'
|
||
record(label, '%s; ready за %.0f с, попыток %s, варианты = контрольным' % (how, time.monotonic() - started, attempts))
|
||
|
||
# S11. Зависший worker: пауза 35 с > аренды 25 с. Работу забирает другая попытка, поздний результат отброшен.
|
||
started = time.monotonic()
|
||
s11 = upload('failpoint-worker_after_side_effect-35s.png')
|
||
st = settle(s11, 90)
|
||
j = job(s11)
|
||
ready_after = time.monotonic() - started
|
||
wait_for('stale attempt fenced', lambda: any('аренда потеряна' in line for line in deliveries(j[0])), 60)
|
||
check(st[0] == 'ready' and j[1:3] == ['done', '2'] and variants(s11, 1) == ref, 'S11: %s %s' % (st, j))
|
||
record('S11 worker завис на 35 с (аренда 25 с)', 'попытка 2 → ready за %.0f с; попытка 1 проснулась → fencing: результат отброшен' % ready_after)
|
||
|
||
# S12. S3 недоступен для worker: backoff с jitter → retry_exhausted → реестр failed → replay.
|
||
compose('stop', 'worker')
|
||
s12 = upload('transient.png')
|
||
compose('stop', 's3')
|
||
try:
|
||
compose('up', '-d', '--no-deps', 'worker', env=TEST_ENV) # start поднял бы и S3 по цепочке depends_on
|
||
st = wait_for('retry exhausted', lambda: state(s12)[0] == 'failed' and state(s12), 120)
|
||
j = job(s12)
|
||
delays = rows("SELECT attempt, round(extract(epoch FROM available_at - created_at)::numeric, 1) FROM outbox "
|
||
"WHERE job_id = '%s' AND attempt > 1 ORDER BY attempt" % j[0])
|
||
registry = [r for r in jobs('failed') if r.get('attachment_id') == s12]
|
||
meta = c.call('GET', '/api/v1/attachments/' + s12, 200, actor=bob, schema='Attachment')['json']
|
||
c.call('GET', '/api/v1/attachments/' + s12 + '/content', 409, actor=bob)
|
||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [s12]},
|
||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||
check(st[2] == 'retry_exhausted' and j[1:3] == ['failed', '5'] and len(registry) == 1
|
||
and meta['status'] == 'failed' and meta['error_code'] == 'retry_exhausted', 'S12: %s %s' % (st, j))
|
||
finally:
|
||
compose('start', 's3')
|
||
wait_for('s3 healthy', lambda: healthy('s3'), 60)
|
||
replay = jobs('replay', s12)[-1]
|
||
st = settle(s12)
|
||
audit = rows("SELECT previous_error, previous_attempts FROM job_replays WHERE job_id = '%s'" % j[0])
|
||
again = jobs('replay', s12)[-1]
|
||
check(replay['replayed'] and st[0] == 'ready' and audit == [['retry_exhausted', '5']] and not again['replayed']
|
||
and variants(s12, 1) == ref, 'S12 replay: %s %s %s' % (replay, st, again))
|
||
record('S12 S3 недоступен → retry_exhausted', 'задержки попыток 2–5: %s с; failed в реестре, API: failed/409'
|
||
% ', '.join(d for _, d in delays))
|
||
record(' replay', 'аудит (retry_exhausted, 5) → ready; повторный replay: %s' % again['reason'])
|
||
|
||
# S13. Постоянная ошибка: файл проходит приём, но не декодируется — failed сразу, без повторов.
|
||
s13 = upload('bad-filter.png', bad_filter_png())
|
||
st = wait_for('permanent failure', lambda: state(s13)[0] == 'failed' and state(s13), 60)
|
||
check(st[2] == 'invalid_image' and job(s13)[1:3] == ['failed', '1'], 'S13: %s' % st)
|
||
record('S13 постоянная ошибка (фильтр PNG = 9)', 'failed invalid_image с первой попытки, без повторов')
|
||
|
||
# S14. Брокер остановлен: API принимает файлы (202), задания ждут в outbox; брокер вернулся — ready.
|
||
compose('stop', 'rabbitmq')
|
||
try:
|
||
s14 = [upload('broker-down-%d.png' % i) for i in range(3)]
|
||
c.call('GET', '/health/ready', 200) # readiness API не зависит от брокера
|
||
unpublished = sql("SELECT count(*) FROM outbox o JOIN jobs j ON j.job_id = o.job_id WHERE o.published_at IS NULL "
|
||
"AND j.attachment_id IN ('%s')" % "', '".join(s14))
|
||
check(unpublished == '3', 'S14: unpublished %s' % unpublished)
|
||
finally:
|
||
compose('start', 'rabbitmq')
|
||
started = time.monotonic()
|
||
for aid in s14:
|
||
check(settle(aid, 180)[0] == 'ready' and variants(aid, 1) == ref, 'S14: %s' % aid)
|
||
record('S14 брокер остановлен', '3 upload → 202, неопубликовано в outbox 3; брокер вернулся → все ready за %.0f с'
|
||
% (time.monotonic() - started))
|
||
|
||
# Инварианты по всем вложениям тестового чата.
|
||
print('\nИнварианты:')
|
||
atts = rows("SELECT id, state, coalesce(pipeline_version, 0), filename FROM attachments WHERE chat_id = '%s'" % chat['id'])
|
||
open_work = [a for a in atts if a[1] not in ('ready', 'failed') and a[0] != s1]
|
||
check(not open_work, 'I1: %s' % open_work)
|
||
print(' I1 нет потерянной работы: %d вложений с заданием, все ready/failed; S1 (без 202) — pending до reconciler' % (len(atts) - 1))
|
||
counts = rows("SELECT v.attachment_id, v.pipeline_version, count(*) FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id "
|
||
"WHERE a.chat_id = '%s' GROUP BY 1, 2" % chat['id'])
|
||
check(all(n == '3' for _, _, n in counts), 'I2: %s' % counts)
|
||
ready = [a for a in atts if a[1] == 'ready']
|
||
check(all(any(aid == a[0] and ver == a[2] for aid, ver, _ in counts) for a in ready), 'I2: ready without variants')
|
||
print(' I2 один результат: %d наборов (вложение, версия), в каждом ровно 3 варианта' % len(counts))
|
||
check(state(s4)[:2] == ['ready', '2'] and variants(s4, 2) == v2 and state(s7)[:2] == ['ready', '2'] and variants(s7, 2) == v2_7, 'I3')
|
||
print(' I3 v2 не перезаписана: S5–S7 остались v2, sha256 вариантов v2 не изменились после старых job')
|
||
same = [a for a in ready if a[2] == '1']
|
||
check(all(variants(a[0], 1) == ref for a in same) and variants(s4, 1) == ref, 'I4')
|
||
print(' I4 тот же результат: у %d ready v1 после сбоев sha256 вариантов = контрольным' % len(same))
|
||
downloads = 0
|
||
for a in ready:
|
||
meta = c.call('GET', '/api/v1/attachments/' + a[0], 200, actor=bob, schema='Attachment')['json']
|
||
check(c.call('GET', '/api/v1/attachments/' + a[0] + '/content', 200, actor=bob)['bytes'] == PNG, 'I5 original')
|
||
for v in meta['variants']:
|
||
body = c.call('GET', '/api/v1/attachments/%s/content?variant=%s' % (a[0], v['name']), 200, actor=bob)['bytes']
|
||
check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], 'I5 %s' % v['name'])
|
||
downloads += 1
|
||
print(' I5 API согласован: %d оригиналов и %d вариантов скачаны, размер и sha256 = метаданным' % (len(ready), downloads))
|
||
states = dict((s, n) for s, n in rows("SELECT j.state, count(*) FROM jobs j JOIN attachments a ON a.id = j.attachment_id "
|
||
"WHERE a.chat_id = '%s' GROUP BY 1" % chat['id']))
|
||
check(set(states) <= {'done', 'failed', 'superseded'}, 'I6 jobs: %s' % states)
|
||
# Остатки ожидаются ровно от трёх сценариев; находки по чужим вложениям (прерванные прошлые прогоны) не считаем.
|
||
found, summary = reconcile('--grace=0')
|
||
ids = {a[0] for a in atts}
|
||
mine = {}
|
||
for f in found:
|
||
owner = f.get('id') or f['key'].split('/')[1]
|
||
if owner in ids:
|
||
mine.setdefault(f['finding'], set()).add(owner)
|
||
check(mine == {'stale_pending': {s1}, 'orphan_variant': {crashed['worker_after_side_effect']},
|
||
'published_staging': {crashed['worker_after_commit']}}, 'I6 reconcile: %s' % mine)
|
||
n_variants = sum(1 for f in found if f['finding'] == 'orphan_variant' and f['key'].split('/')[1] in ids)
|
||
fixed = reconcile('--grace=0', '--apply')[1]
|
||
after = reconcile('--grace=0')[1]['findings']
|
||
check(not after and n_variants == 3, 'I6 after cleanup: %s, orphan variants %d' % (after, n_variants))
|
||
print(' I6 job: %s; reconciler: stale_pending S1, orphan_variant ×%d попытки 1 в S9, published_staging S10 — и только они;'
|
||
' применено %d (всего на стенде %s), повторный обход: {}' % (json.dumps(states), n_variants, fixed['applied'],
|
||
json.dumps(summary['findings'])))
|
||
finally:
|
||
compose('start', 's3', 'worker')
|
||
compose('up', '-d', '--no-deps', 'api', 'worker') # FAILPOINTS и таймаут — снова из .env
|
||
wait_for('api healthy', lambda: healthy('api'), 90)
|
||
print('\n'.join(['', 'Сводка:'] + report))
|
||
print('PASS: 14 fault scenarios, invariants I1–I6 hold (at-least-once delivery + idempotent effect, not exactly-once)')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|