Files
mtusi2026lab6/scripts/lab5/fault_suite.py
T
ilya.tyunin 6889e77ab4 lab6: базовый коммит из lab5 (0790599)
Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.

Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.

Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
  mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
  infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
  от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
  (OTEL_*) из skeleton.
2026-11-09 16:44:21 +03:00

750 lines
29 KiB
Python
Executable File

#!/usr/bin/env python3
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
import argparse
import json
import re
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
from stand import ( # noqa: E402
ROOT,
Failure,
check,
clear_fault,
clear_faults,
cli,
compose,
docker,
new_client,
rabbit_queues,
sample_bytes,
set_fault,
sha256,
upload,
variant_sha,
wait_for,
wait_ready,
worker_env,
worker_start_count,
)
from pngkit import worst_decode_png # noqa: E402
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
'make stand-up')
LEVEL4 = [
'crash_after_commit_before_publish',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'retry_and_replay',
'hung_worker',
'poison',
'big_image',
]
TRACK2 = [
'duplicate',
'reorder',
'crash_before_side_effect',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'relay_crash_after_publish',
'stale_version',
]
class Tracker(threading.Thread):
"""Polls metadata while a scenario runs to prove readiness never regresses."""
def __init__(self, base_url, actor, ca_file=None):
super().__init__(daemon=True)
self.client = new_client(base_url, ca_file)
self.actor = actor
self.ids = []
self.records = {}
self.stop_event = threading.Event()
def track(self, attachment_id):
if attachment_id not in self.records:
self.ids.append(attachment_id)
self.records[attachment_id] = []
def run(self):
while not self.stop_event.is_set():
for attachment_id in list(self.ids):
try:
meta = self.client.call(
'GET', '/api/v1/attachments/' + attachment_id,
actor=self.actor, schema='Attachment',
)['json']
self.records[attachment_id].append(
(time.monotonic(), meta['status'],
meta.get('ready_pipeline_version'))
)
except Exception:
pass
self.stop_event.wait(0.2)
def stop(self):
self.stop_event.set()
self.join(timeout=5)
def build_message(status, job, attempt=None):
created = next(
event for event in status['events']
if event['job_id'] == job['job_id'] and event['event'] == 'created'
)
return {
'schema_version': 1,
'job_id': job['job_id'],
'type': 'attachment.process.v1',
'attachment_id': status['attachment']['id'],
'pipeline_version': job['pipeline_version'],
'attempt': attempt if attempt is not None else job['attempt'],
'created_at': created['created_at'],
}
def publish_raw(messages):
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
for message in messages)
code, payload = cli('publish-raw', stdin=text)
check(code == 0, 'publish-raw failed: ' + str(payload))
return payload
def status_of(attachment_id):
code, payload = cli('status', attachment_id)
check(code == 0, 'status failed: ' + str(payload))
return payload
def flush_outbox(attachment_id):
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
30, 'outbox publish', 1.0)
def new_attachment_id(chat_id, before):
rows = cli('find-by-chat', chat_id)[1]
for row in reversed(rows):
if row['id'] not in before:
return row['id']
return None
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
compare_baseline=True):
if attachment_id is None:
return {'I1': True, 'I2': True, 'I3': True,
'I4': True, 'I5': True, 'I6': True}
status = status_of(attachment_id)
attachment = status['attachment']
jobs = status['jobs']
variants = status['variants']
ready_version = attachment['ready_pipeline_version']
records = tracker.records.get(attachment_id, [])
reached = any(record[1] == ('failed' if expect_failed else 'ready')
for record in records)
if not records:
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
i1 = reached or attachment['status'] in ('ready', 'failed')
ready_variants = [v for v in variants
if v['pipeline_version'] == ready_version]
succeeded_ids = {job['job_id'] for job in jobs
if job['status'] == 'succeeded'}
succeeded_events = {}
for event in status['events']:
if event['event'] == 'succeeded':
succeeded_events[event['job_id']] = (
succeeded_events.get(event['job_id'], 0) + 1
)
i2 = (len(ready_variants) == 3
and all(succeeded_events.get(job_id) == 1
for job_id in succeeded_ids))
i3 = True
seen_ready = False
last_version = 0
for _at, state, version in records:
if seen_ready and state != 'ready':
i3 = False
if state == 'ready':
seen_ready = True
if version is not None:
if version < last_version:
i3 = False
last_version = version
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
keys = {(variant['pipeline_version'], variant['variant'])
for variant in variants}
i5 = len(keys) == len(variants)
if compare_baseline and ready_version == 1 and ready_variants:
observed = {variant['variant']: variant['sha256']
for variant in ready_variants}
i6 = all(observed.get(name) == digest
for name, digest in baseline.items())
else:
i6 = True
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
'I5': i5, 'I6': i6}
# --- scenarios -----------------------------------------------------------
def scenario_duplicate(ctx):
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'duplicate.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
status = status_of(attachment['id'])
before = {variant['variant']: variant['s3']
for variant in status['variants']}
message = build_message(status, status['jobs'][0])
publish_raw([message] * 5)
wait_for(
lambda: len([event for event in status_of(attachment['id'])['events']
if event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded']) == 5,
20, 'five duplicate_skipped events', 1.0,
)
after_status = status_of(attachment['id'])
after = {variant['variant']: variant['s3']
for variant in after_status['variants']}
check(before == after, 'duplicate delivery rewrote variant objects')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
def scenario_reorder(ctx):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'reorder.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
flush_outbox(attachment['id'])
status = status_of(attachment['id'])
message = build_message(status, status['jobs'][0])
stale = dict(message, attempt=99)
publish_raw([stale, message])
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
finally:
compose('start', 'worker')
after_status = status_of(attachment['id'])
skipped = [event for event in after_status['events']
if event['event'] == 'duplicate_skipped']
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
check(len([event for event in after_status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one successful effect')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
def crash_scenario(ctx, point, filename, expected_attempt,
expect_lease_expired=False, expect_duplicate=False):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), filename)
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
starts = worker_start_count()
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
finally:
compose('start', 'worker')
status = status_of(attachment['id'])
job = status['jobs'][0]
check(job['attempt'] == expected_attempt,
'expected attempt ' + str(expected_attempt) +
', got ' + str(job['attempt']))
check(worker_start_count() >= starts + 2,
'worker process did not restart after the injected crash')
if expect_lease_expired:
check(any(event['event'] == 'lease_expired'
for event in status['events']),
'missing lease_expired event')
if expect_duplicate:
check(any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded'
for event in status['events']),
'missing already_succeeded duplicate after redelivery')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt']],
'notes': 'crashed at ' + point}
def scenario_crash_before_side_effect(ctx):
return crash_scenario(ctx, 'worker.before_side_effect',
'crash-before.png', 2, expect_lease_expired=True)
def scenario_crash_after_side_effect(ctx):
return crash_scenario(ctx, 'worker.after_side_effect',
'crash-after.png', 2, expect_lease_expired=True)
def scenario_crash_after_commit_before_ack(ctx):
return crash_scenario(ctx, 'worker.after_commit_before_ack',
'crash-ack.png', 1, expect_duplicate=True)
def scenario_crash_after_commit_before_publish(ctx):
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
set_fault('fault:api.after_commit_before_publish', 'crash')
try:
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=crash-publish.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# A killed replica may close the connection instead of answering.
pass
finally:
clear_fault('fault:api.after_commit_before_publish')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after API crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'created']) == 1,
'expected exactly one created event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'outbox relay published the committed job after API restart'}
def scenario_relay_crash_after_publish(ctx):
compose('stop', 'worker')
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
try:
set_fault('fault:relay.after_publish_before_mark', 'crash')
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=relay-crash.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# The relay may crash the handling replica before the reply flushes.
pass
clear_fault('fault:relay.after_publish_before_mark')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after relay crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
60, 'outbox published after relay crash', 1.0)
queue_depth = rabbit_queues().get('attachments.process', {}).get(
'messages', 0)
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
clear_fault('fault:relay.after_publish_before_mark')
compose('start', 'worker')
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one succeeded event')
duplicated = any(event['event'] == 'duplicate_skipped'
for event in status['events'])
note = ('relay republished the same message; duplicate absorbed'
if duplicated else
'relay published once in this run; I1/I2 still hold')
if queue_depth >= 2:
check(duplicated, 'two queued copies but no duplicate_skipped event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': note}
def scenario_stale_version(ctx):
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
'stale_version needs WORKER_CONCURRENCY>=2')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'stale.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
flush_outbox(attachment_id)
code, payload = cli('reprocess', attachment_id,
'--pipeline-version', '2', '--reason', 'suite')
check(code == 0, 'reprocess v2 failed: ' + str(payload))
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:8000')
compose('start', 'worker')
v2_etags = None
def both_terminal():
nonlocal v2_etags
status = status_of(attachment_id)
jobs = {(job['pipeline_version']): job for job in status['jobs']}
if '2' in jobs or 2 in jobs:
v2_job = jobs.get(2)
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
v2_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
v1_job = jobs.get(1)
v2_job = jobs.get(2)
return (v1_job and v2_job
and v1_job['status'] == 'superseded'
and v2_job['status'] == 'succeeded')
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
status = status_of(attachment_id)
check(status['attachment']['ready_pipeline_version'] == 2,
'ready pipeline version is not 2')
check(all(variant['pipeline_version'] == 2
for variant in status['variants']),
'variant rows from v1 survived')
after_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
served = ctx['client'].call(
'GET',
'/api/v1/attachments/' + attachment_id +
'/content?variant=thumbnail',
actor=ctx['alice'],
)
v2_thumbnail = next(variant for variant in status['variants']
if variant['variant'] == 'thumbnail')
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
'served thumbnail is not the v2 object')
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
'served thumbnail equals the v1 baseline')
v1_job = next(job for job in status['jobs']
if job['pipeline_version'] == 1)
publish_raw([build_message(status, v1_job)])
wait_for(
lambda: any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'superseded'
for event in status_of(attachment_id)['events']),
20, 'superseded duplicate skip', 1.0,
)
finally:
clear_fault('fault:worker.after_side_effect:' +
(attachment['id'] if 'attachment' in locals() else ''))
compose('start', 'worker')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'v1 finished after v2 and was recorded as superseded'}
def scenario_poison(ctx):
before = rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0)
publish_raw(['not-json', '{"schema_version":1}'])
wait_for(
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0) >= before + 2,
15, 'two poison messages in DLQ', 0.5,
)
return {'attachment_id': None, 'job_attempts': [],
'notes': 'malformed messages moved to the DLQ without touching jobs'}
def scenario_retry_and_replay(ctx):
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'retry.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
'transient')
compose('start', 'worker')
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
90, allow_failed=True)
finally:
compose('start', 'worker')
check(failed['status'] == 'failed', 'transient fault was not exhausted')
check(failed['error_code'] == 'retry_exhausted',
'expected retry_exhausted, got ' + str(failed['error_code']))
status = status_of(attachment_id)
retries = [event for event in status['events']
if event['event'] == 'retry_scheduled']
check(len(retries) == max_attempts - 1,
'expected ' + str(max_attempts - 1) + ' retries, got ' +
str(len(retries)))
for index, event in enumerate(retries):
limit = min(cap, base * 2 ** index)
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
'retry delay exceeds its jitter cap')
ctx['client'].call(
'GET', '/api/v1/attachments/' + attachment_id + '/content',
409, actor=ctx['alice'],
)
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
check(code == 3 and payload.get('reason') in ('no_failed_job',
'already_ready'),
'replay of a ready attachment was not refused')
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
code, payload = cli('replay', attachment_id, '--reason', 'suite')
check(code == 0, 'operator replay failed: ' + str(payload))
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
status = status_of(attachment_id)
replayed = [event for event in status['events']
if event['event'] == 'replayed']
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
'missing replayed event or operator audit')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'transient injected fault exhausted retries; replay recovered'}
def scenario_hung_worker(ctx):
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'hung.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:' + str((timeout_seconds + 5) * 1000))
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
compose('start', 'worker')
status = status_of(attachment_id)
check(any(event['event'] == 'retry_scheduled'
and event['detail'].get('code') == 'processing_timeout'
for event in status['events']),
'missing processing_timeout retry')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'hard timeout aborted the hung render and retried'}
def scenario_big_image(ctx):
ctx['compare_baseline'] = False
data = worst_decode_png()
containers = compose('ps', '-q', 'worker').stdout.split()
stop = threading.Event()
peak = {'mb': 0.0}
def sample():
while not stop.is_set():
if containers:
result = docker(
'stats', '--no-stream', '--format',
'{{.Name}} {{.MemUsage}}', *containers,
check_result=False,
)
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) < 2:
continue
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
if not match:
continue
value = float(match.group(1))
if match.group(2) == 'GiB':
value *= 1024
elif match.group(2) == 'KiB':
value /= 1024
peak['mb'] = max(peak['mb'], value)
stop.wait(0.3)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
try:
def do_upload(index):
client = new_client(ctx['base_url'])
return upload(client, ctx['alice'], ctx['chat'], data,
'big-' + str(index) + '.png')
with ThreadPoolExecutor(max_workers=2) as pool:
attachments = list(pool.map(do_upload, (1, 2)))
ctx['attachment_id'] = attachments[0]['id']
for attachment in attachments:
ctx['tracker'].track(attachment['id'])
results = [wait_ready(ctx['client'], ctx['alice'],
attachment['id'], 120, allow_failed=True)
for attachment in attachments]
finally:
stop.set()
sampler.join(timeout=5)
for meta in results:
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
if meta['status'] == 'failed':
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
'unexpected big-image error code')
for container in containers:
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
check_result=False)
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
return {'attachment_id': attachments[0]['id'],
'job_attempts': [],
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
SCENARIOS = {
'duplicate': scenario_duplicate,
'reorder': scenario_reorder,
'crash_before_side_effect': scenario_crash_before_side_effect,
'crash_after_side_effect': scenario_crash_after_side_effect,
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
'relay_crash_after_publish': scenario_relay_crash_after_publish,
'stale_version': scenario_stale_version,
'poison': scenario_poison,
'retry_and_replay': scenario_retry_and_replay,
'hung_worker': scenario_hung_worker,
'big_image': scenario_big_image,
}
def select(name):
if name == 'all':
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
if name == 'level4':
return LEVEL4
if name == 'track2':
return TRACK2
if name in SCENARIOS:
return [name]
raise SystemExit('unknown scenario: ' + name)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=None)
parser.add_argument('--ca-file', default=None)
parser.add_argument('--scenario', default='level4')
parser.add_argument('--repeat', type=int, default=1)
parser.add_argument('--output', default=None)
args = parser.parse_args()
if worker_env('FAULT_INJECTION_ENABLED') != '1':
print('fault injection is disabled; restart the stand with:\n ' + HINT,
file=sys.stderr)
raise SystemExit(2)
client = new_client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
clear_faults()
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
ready = wait_ready(client, alice, reference['id'], 60)
baseline = variant_sha(ready)
selected = select(args.scenario)
results = []
failed = 0
for name in selected:
for repeat in range(1, args.repeat + 1):
clear_faults()
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
args.ca_file)
tracker.start()
started = time.monotonic()
ctx = {
'client': client,
'base_url': args.base_url or 'http://localhost:8080',
'alice': alice,
'chat': chat,
'tracker': tracker,
'baseline': baseline,
'reference_id': reference['id'],
'ca_file': args.ca_file,
}
scenario = {'attachment_id': None, 'job_attempts': [],
'notes': ''}
passed = True
try:
scenario.update(SCENARIOS[name](ctx))
except Exception as error: # noqa: BLE001 - scenario boundary
passed = False
scenario['notes'] = type(error).__name__ + ': ' + str(error)
finally:
tracker.stop()
clear_faults()
compose('start', 'worker', check_result=False)
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
try:
invariants = evaluate(
tracker, attachment_id, baseline,
compare_baseline=ctx.get('compare_baseline', True),
)
except Exception: # noqa: BLE001
invariants = {key: False for key in
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
passed = passed and all(invariants.values())
if not passed:
failed += 1
result = {
'name': name,
'repeat': repeat,
'passed': passed,
'duration_s': round(time.monotonic() - started, 3),
'attachment_id': attachment_id,
'job_attempts': scenario['job_attempts'],
'events_summary': scenario.get('events_summary', {}),
'invariants': invariants,
'notes': scenario['notes'],
}
results.append(result)
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
' ' + str(result['duration_s']) + 's ' +
json.dumps(invariants) + ' ' + result['notes'],
flush=True)
if args.output:
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
raise SystemExit(1 if failed else 0)
if __name__ == '__main__':
main()