forked from mtusi2026/mtusi2026lab6
Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.
Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.
Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
(OTEL_*) из skeleton.
750 lines
29 KiB
Python
Executable File
750 lines
29 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
|
|
|
|
from stand import ( # noqa: E402
|
|
ROOT,
|
|
Failure,
|
|
check,
|
|
clear_fault,
|
|
clear_faults,
|
|
cli,
|
|
compose,
|
|
docker,
|
|
new_client,
|
|
rabbit_queues,
|
|
sample_bytes,
|
|
set_fault,
|
|
sha256,
|
|
upload,
|
|
variant_sha,
|
|
wait_for,
|
|
wait_ready,
|
|
worker_env,
|
|
worker_start_count,
|
|
)
|
|
from pngkit import worst_decode_png # noqa: E402
|
|
|
|
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
|
|
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
|
|
'make stand-up')
|
|
|
|
LEVEL4 = [
|
|
'crash_after_commit_before_publish',
|
|
'crash_after_side_effect',
|
|
'crash_after_commit_before_ack',
|
|
'retry_and_replay',
|
|
'hung_worker',
|
|
'poison',
|
|
'big_image',
|
|
]
|
|
TRACK2 = [
|
|
'duplicate',
|
|
'reorder',
|
|
'crash_before_side_effect',
|
|
'crash_after_side_effect',
|
|
'crash_after_commit_before_ack',
|
|
'relay_crash_after_publish',
|
|
'stale_version',
|
|
]
|
|
|
|
|
|
class Tracker(threading.Thread):
|
|
"""Polls metadata while a scenario runs to prove readiness never regresses."""
|
|
|
|
def __init__(self, base_url, actor, ca_file=None):
|
|
super().__init__(daemon=True)
|
|
self.client = new_client(base_url, ca_file)
|
|
self.actor = actor
|
|
self.ids = []
|
|
self.records = {}
|
|
self.stop_event = threading.Event()
|
|
|
|
def track(self, attachment_id):
|
|
if attachment_id not in self.records:
|
|
self.ids.append(attachment_id)
|
|
self.records[attachment_id] = []
|
|
|
|
def run(self):
|
|
while not self.stop_event.is_set():
|
|
for attachment_id in list(self.ids):
|
|
try:
|
|
meta = self.client.call(
|
|
'GET', '/api/v1/attachments/' + attachment_id,
|
|
actor=self.actor, schema='Attachment',
|
|
)['json']
|
|
self.records[attachment_id].append(
|
|
(time.monotonic(), meta['status'],
|
|
meta.get('ready_pipeline_version'))
|
|
)
|
|
except Exception:
|
|
pass
|
|
self.stop_event.wait(0.2)
|
|
|
|
def stop(self):
|
|
self.stop_event.set()
|
|
self.join(timeout=5)
|
|
|
|
|
|
def build_message(status, job, attempt=None):
|
|
created = next(
|
|
event for event in status['events']
|
|
if event['job_id'] == job['job_id'] and event['event'] == 'created'
|
|
)
|
|
return {
|
|
'schema_version': 1,
|
|
'job_id': job['job_id'],
|
|
'type': 'attachment.process.v1',
|
|
'attachment_id': status['attachment']['id'],
|
|
'pipeline_version': job['pipeline_version'],
|
|
'attempt': attempt if attempt is not None else job['attempt'],
|
|
'created_at': created['created_at'],
|
|
}
|
|
|
|
|
|
def publish_raw(messages):
|
|
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
|
|
for message in messages)
|
|
code, payload = cli('publish-raw', stdin=text)
|
|
check(code == 0, 'publish-raw failed: ' + str(payload))
|
|
return payload
|
|
|
|
|
|
def status_of(attachment_id):
|
|
code, payload = cli('status', attachment_id)
|
|
check(code == 0, 'status failed: ' + str(payload))
|
|
return payload
|
|
|
|
|
|
def flush_outbox(attachment_id):
|
|
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
|
30, 'outbox publish', 1.0)
|
|
|
|
|
|
def new_attachment_id(chat_id, before):
|
|
rows = cli('find-by-chat', chat_id)[1]
|
|
for row in reversed(rows):
|
|
if row['id'] not in before:
|
|
return row['id']
|
|
return None
|
|
|
|
|
|
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
|
|
compare_baseline=True):
|
|
if attachment_id is None:
|
|
return {'I1': True, 'I2': True, 'I3': True,
|
|
'I4': True, 'I5': True, 'I6': True}
|
|
status = status_of(attachment_id)
|
|
attachment = status['attachment']
|
|
jobs = status['jobs']
|
|
variants = status['variants']
|
|
ready_version = attachment['ready_pipeline_version']
|
|
|
|
records = tracker.records.get(attachment_id, [])
|
|
reached = any(record[1] == ('failed' if expect_failed else 'ready')
|
|
for record in records)
|
|
if not records:
|
|
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
|
|
i1 = reached or attachment['status'] in ('ready', 'failed')
|
|
|
|
ready_variants = [v for v in variants
|
|
if v['pipeline_version'] == ready_version]
|
|
succeeded_ids = {job['job_id'] for job in jobs
|
|
if job['status'] == 'succeeded'}
|
|
succeeded_events = {}
|
|
for event in status['events']:
|
|
if event['event'] == 'succeeded':
|
|
succeeded_events[event['job_id']] = (
|
|
succeeded_events.get(event['job_id'], 0) + 1
|
|
)
|
|
i2 = (len(ready_variants) == 3
|
|
and all(succeeded_events.get(job_id) == 1
|
|
for job_id in succeeded_ids))
|
|
|
|
i3 = True
|
|
seen_ready = False
|
|
last_version = 0
|
|
for _at, state, version in records:
|
|
if seen_ready and state != 'ready':
|
|
i3 = False
|
|
if state == 'ready':
|
|
seen_ready = True
|
|
if version is not None:
|
|
if version < last_version:
|
|
i3 = False
|
|
last_version = version
|
|
|
|
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
|
|
keys = {(variant['pipeline_version'], variant['variant'])
|
|
for variant in variants}
|
|
i5 = len(keys) == len(variants)
|
|
|
|
if compare_baseline and ready_version == 1 and ready_variants:
|
|
observed = {variant['variant']: variant['sha256']
|
|
for variant in ready_variants}
|
|
i6 = all(observed.get(name) == digest
|
|
for name, digest in baseline.items())
|
|
else:
|
|
i6 = True
|
|
|
|
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
|
|
'I5': i5, 'I6': i6}
|
|
|
|
|
|
# --- scenarios -----------------------------------------------------------
|
|
|
|
|
|
def scenario_duplicate(ctx):
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), 'duplicate.png')
|
|
ctx['attachment_id'] = attachment['id']
|
|
ctx['tracker'].track(attachment['id'])
|
|
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
|
status = status_of(attachment['id'])
|
|
before = {variant['variant']: variant['s3']
|
|
for variant in status['variants']}
|
|
message = build_message(status, status['jobs'][0])
|
|
publish_raw([message] * 5)
|
|
wait_for(
|
|
lambda: len([event for event in status_of(attachment['id'])['events']
|
|
if event['event'] == 'duplicate_skipped'
|
|
and event['detail'].get('reason') == 'already_succeeded']) == 5,
|
|
20, 'five duplicate_skipped events', 1.0,
|
|
)
|
|
after_status = status_of(attachment['id'])
|
|
after = {variant['variant']: variant['s3']
|
|
for variant in after_status['variants']}
|
|
check(before == after, 'duplicate delivery rewrote variant objects')
|
|
return {'attachment_id': attachment['id'],
|
|
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
|
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
|
|
|
|
|
|
def scenario_reorder(ctx):
|
|
compose('stop', 'worker')
|
|
try:
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), 'reorder.png')
|
|
ctx['attachment_id'] = attachment['id']
|
|
ctx['tracker'].track(attachment['id'])
|
|
flush_outbox(attachment['id'])
|
|
status = status_of(attachment['id'])
|
|
message = build_message(status, status['jobs'][0])
|
|
stale = dict(message, attempt=99)
|
|
publish_raw([stale, message])
|
|
compose('start', 'worker')
|
|
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
|
finally:
|
|
compose('start', 'worker')
|
|
after_status = status_of(attachment['id'])
|
|
skipped = [event for event in after_status['events']
|
|
if event['event'] == 'duplicate_skipped']
|
|
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
|
|
check(len([event for event in after_status['events']
|
|
if event['event'] == 'succeeded']) == 1,
|
|
'expected exactly one successful effect')
|
|
return {'attachment_id': attachment['id'],
|
|
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
|
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
|
|
|
|
|
|
def crash_scenario(ctx, point, filename, expected_attempt,
|
|
expect_lease_expired=False, expect_duplicate=False):
|
|
compose('stop', 'worker')
|
|
try:
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), filename)
|
|
ctx['attachment_id'] = attachment['id']
|
|
ctx['tracker'].track(attachment['id'])
|
|
starts = worker_start_count()
|
|
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
|
|
compose('start', 'worker')
|
|
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
|
|
finally:
|
|
compose('start', 'worker')
|
|
status = status_of(attachment['id'])
|
|
job = status['jobs'][0]
|
|
check(job['attempt'] == expected_attempt,
|
|
'expected attempt ' + str(expected_attempt) +
|
|
', got ' + str(job['attempt']))
|
|
check(worker_start_count() >= starts + 2,
|
|
'worker process did not restart after the injected crash')
|
|
if expect_lease_expired:
|
|
check(any(event['event'] == 'lease_expired'
|
|
for event in status['events']),
|
|
'missing lease_expired event')
|
|
if expect_duplicate:
|
|
check(any(event['event'] == 'duplicate_skipped'
|
|
and event['detail'].get('reason') == 'already_succeeded'
|
|
for event in status['events']),
|
|
'missing already_succeeded duplicate after redelivery')
|
|
return {'attachment_id': attachment['id'],
|
|
'job_attempts': [job['attempt']],
|
|
'notes': 'crashed at ' + point}
|
|
|
|
|
|
def scenario_crash_before_side_effect(ctx):
|
|
return crash_scenario(ctx, 'worker.before_side_effect',
|
|
'crash-before.png', 2, expect_lease_expired=True)
|
|
|
|
|
|
def scenario_crash_after_side_effect(ctx):
|
|
return crash_scenario(ctx, 'worker.after_side_effect',
|
|
'crash-after.png', 2, expect_lease_expired=True)
|
|
|
|
|
|
def scenario_crash_after_commit_before_ack(ctx):
|
|
return crash_scenario(ctx, 'worker.after_commit_before_ack',
|
|
'crash-ack.png', 1, expect_duplicate=True)
|
|
|
|
|
|
def scenario_crash_after_commit_before_publish(ctx):
|
|
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
|
set_fault('fault:api.after_commit_before_publish', 'crash')
|
|
try:
|
|
try:
|
|
ctx['client'].call(
|
|
'POST',
|
|
'/api/v1/chats/' + ctx['chat']['id'] +
|
|
'/attachments?filename=crash-publish.png',
|
|
(202, 502, 503, 504),
|
|
actor=ctx['alice'],
|
|
raw=sample_bytes(),
|
|
headers={'Content-Type': 'image/png'},
|
|
)
|
|
except Failure:
|
|
# A killed replica may close the connection instead of answering.
|
|
pass
|
|
finally:
|
|
clear_fault('fault:api.after_commit_before_publish')
|
|
attachment_id = wait_for(
|
|
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
|
'attachment row after API crash', 1.0,
|
|
)
|
|
ctx['attachment_id'] = attachment_id
|
|
ctx['tracker'].track(attachment_id)
|
|
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
|
status = status_of(attachment_id)
|
|
check(len([event for event in status['events']
|
|
if event['event'] == 'created']) == 1,
|
|
'expected exactly one created event')
|
|
return {'attachment_id': attachment_id,
|
|
'job_attempts': [job['attempt'] for job in status['jobs']],
|
|
'notes': 'outbox relay published the committed job after API restart'}
|
|
|
|
|
|
def scenario_relay_crash_after_publish(ctx):
|
|
compose('stop', 'worker')
|
|
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
|
try:
|
|
set_fault('fault:relay.after_publish_before_mark', 'crash')
|
|
try:
|
|
ctx['client'].call(
|
|
'POST',
|
|
'/api/v1/chats/' + ctx['chat']['id'] +
|
|
'/attachments?filename=relay-crash.png',
|
|
(202, 502, 503, 504),
|
|
actor=ctx['alice'],
|
|
raw=sample_bytes(),
|
|
headers={'Content-Type': 'image/png'},
|
|
)
|
|
except Failure:
|
|
# The relay may crash the handling replica before the reply flushes.
|
|
pass
|
|
clear_fault('fault:relay.after_publish_before_mark')
|
|
attachment_id = wait_for(
|
|
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
|
'attachment row after relay crash', 1.0,
|
|
)
|
|
ctx['attachment_id'] = attachment_id
|
|
ctx['tracker'].track(attachment_id)
|
|
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
|
60, 'outbox published after relay crash', 1.0)
|
|
queue_depth = rabbit_queues().get('attachments.process', {}).get(
|
|
'messages', 0)
|
|
compose('start', 'worker')
|
|
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
|
finally:
|
|
clear_fault('fault:relay.after_publish_before_mark')
|
|
compose('start', 'worker')
|
|
status = status_of(attachment_id)
|
|
check(len([event for event in status['events']
|
|
if event['event'] == 'succeeded']) == 1,
|
|
'expected exactly one succeeded event')
|
|
duplicated = any(event['event'] == 'duplicate_skipped'
|
|
for event in status['events'])
|
|
note = ('relay republished the same message; duplicate absorbed'
|
|
if duplicated else
|
|
'relay published once in this run; I1/I2 still hold')
|
|
if queue_depth >= 2:
|
|
check(duplicated, 'two queued copies but no duplicate_skipped event')
|
|
return {'attachment_id': attachment_id,
|
|
'job_attempts': [job['attempt'] for job in status['jobs']],
|
|
'notes': note}
|
|
|
|
|
|
def scenario_stale_version(ctx):
|
|
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
|
|
'stale_version needs WORKER_CONCURRENCY>=2')
|
|
compose('stop', 'worker')
|
|
try:
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), 'stale.png')
|
|
attachment_id = attachment['id']
|
|
ctx['attachment_id'] = attachment_id
|
|
ctx['tracker'].track(attachment_id)
|
|
flush_outbox(attachment_id)
|
|
code, payload = cli('reprocess', attachment_id,
|
|
'--pipeline-version', '2', '--reason', 'suite')
|
|
check(code == 0, 'reprocess v2 failed: ' + str(payload))
|
|
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
|
'delay:8000')
|
|
compose('start', 'worker')
|
|
v2_etags = None
|
|
|
|
def both_terminal():
|
|
nonlocal v2_etags
|
|
status = status_of(attachment_id)
|
|
jobs = {(job['pipeline_version']): job for job in status['jobs']}
|
|
if '2' in jobs or 2 in jobs:
|
|
v2_job = jobs.get(2)
|
|
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
|
|
v2_etags = {variant['variant']: variant['s3']
|
|
for variant in status['variants']
|
|
if variant['pipeline_version'] == 2}
|
|
v1_job = jobs.get(1)
|
|
v2_job = jobs.get(2)
|
|
return (v1_job and v2_job
|
|
and v1_job['status'] == 'superseded'
|
|
and v2_job['status'] == 'succeeded')
|
|
|
|
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
|
|
status = status_of(attachment_id)
|
|
check(status['attachment']['ready_pipeline_version'] == 2,
|
|
'ready pipeline version is not 2')
|
|
check(all(variant['pipeline_version'] == 2
|
|
for variant in status['variants']),
|
|
'variant rows from v1 survived')
|
|
after_etags = {variant['variant']: variant['s3']
|
|
for variant in status['variants']
|
|
if variant['pipeline_version'] == 2}
|
|
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
|
|
|
|
served = ctx['client'].call(
|
|
'GET',
|
|
'/api/v1/attachments/' + attachment_id +
|
|
'/content?variant=thumbnail',
|
|
actor=ctx['alice'],
|
|
)
|
|
v2_thumbnail = next(variant for variant in status['variants']
|
|
if variant['variant'] == 'thumbnail')
|
|
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
|
|
'served thumbnail is not the v2 object')
|
|
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
|
|
'served thumbnail equals the v1 baseline')
|
|
|
|
v1_job = next(job for job in status['jobs']
|
|
if job['pipeline_version'] == 1)
|
|
publish_raw([build_message(status, v1_job)])
|
|
wait_for(
|
|
lambda: any(event['event'] == 'duplicate_skipped'
|
|
and event['detail'].get('reason') == 'superseded'
|
|
for event in status_of(attachment_id)['events']),
|
|
20, 'superseded duplicate skip', 1.0,
|
|
)
|
|
finally:
|
|
clear_fault('fault:worker.after_side_effect:' +
|
|
(attachment['id'] if 'attachment' in locals() else ''))
|
|
compose('start', 'worker')
|
|
return {'attachment_id': attachment_id,
|
|
'job_attempts': [job['attempt'] for job in status['jobs']],
|
|
'notes': 'v1 finished after v2 and was recorded as superseded'}
|
|
|
|
|
|
def scenario_poison(ctx):
|
|
before = rabbit_queues().get('attachments.process.dlq', {}).get(
|
|
'messages', 0)
|
|
publish_raw(['not-json', '{"schema_version":1}'])
|
|
wait_for(
|
|
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
|
|
'messages', 0) >= before + 2,
|
|
15, 'two poison messages in DLQ', 0.5,
|
|
)
|
|
return {'attachment_id': None, 'job_attempts': [],
|
|
'notes': 'malformed messages moved to the DLQ without touching jobs'}
|
|
|
|
|
|
def scenario_retry_and_replay(ctx):
|
|
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
|
|
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
|
|
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
|
|
compose('stop', 'worker')
|
|
try:
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), 'retry.png')
|
|
attachment_id = attachment['id']
|
|
ctx['attachment_id'] = attachment_id
|
|
ctx['tracker'].track(attachment_id)
|
|
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
|
|
'transient')
|
|
compose('start', 'worker')
|
|
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
|
|
90, allow_failed=True)
|
|
finally:
|
|
compose('start', 'worker')
|
|
check(failed['status'] == 'failed', 'transient fault was not exhausted')
|
|
check(failed['error_code'] == 'retry_exhausted',
|
|
'expected retry_exhausted, got ' + str(failed['error_code']))
|
|
status = status_of(attachment_id)
|
|
retries = [event for event in status['events']
|
|
if event['event'] == 'retry_scheduled']
|
|
check(len(retries) == max_attempts - 1,
|
|
'expected ' + str(max_attempts - 1) + ' retries, got ' +
|
|
str(len(retries)))
|
|
for index, event in enumerate(retries):
|
|
limit = min(cap, base * 2 ** index)
|
|
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
|
|
'retry delay exceeds its jitter cap')
|
|
ctx['client'].call(
|
|
'GET', '/api/v1/attachments/' + attachment_id + '/content',
|
|
409, actor=ctx['alice'],
|
|
)
|
|
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
|
|
check(code == 3 and payload.get('reason') in ('no_failed_job',
|
|
'already_ready'),
|
|
'replay of a ready attachment was not refused')
|
|
|
|
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
|
|
code, payload = cli('replay', attachment_id, '--reason', 'suite')
|
|
check(code == 0, 'operator replay failed: ' + str(payload))
|
|
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
|
|
status = status_of(attachment_id)
|
|
replayed = [event for event in status['events']
|
|
if event['event'] == 'replayed']
|
|
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
|
|
'missing replayed event or operator audit')
|
|
return {'attachment_id': attachment_id,
|
|
'job_attempts': [job['attempt'] for job in status['jobs']],
|
|
'notes': 'transient injected fault exhausted retries; replay recovered'}
|
|
|
|
|
|
def scenario_hung_worker(ctx):
|
|
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
|
|
compose('stop', 'worker')
|
|
try:
|
|
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
|
sample_bytes(), 'hung.png')
|
|
attachment_id = attachment['id']
|
|
ctx['attachment_id'] = attachment_id
|
|
ctx['tracker'].track(attachment_id)
|
|
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
|
'delay:' + str((timeout_seconds + 5) * 1000))
|
|
compose('start', 'worker')
|
|
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
|
finally:
|
|
compose('start', 'worker')
|
|
status = status_of(attachment_id)
|
|
check(any(event['event'] == 'retry_scheduled'
|
|
and event['detail'].get('code') == 'processing_timeout'
|
|
for event in status['events']),
|
|
'missing processing_timeout retry')
|
|
return {'attachment_id': attachment_id,
|
|
'job_attempts': [job['attempt'] for job in status['jobs']],
|
|
'notes': 'hard timeout aborted the hung render and retried'}
|
|
|
|
|
|
def scenario_big_image(ctx):
|
|
ctx['compare_baseline'] = False
|
|
data = worst_decode_png()
|
|
containers = compose('ps', '-q', 'worker').stdout.split()
|
|
stop = threading.Event()
|
|
peak = {'mb': 0.0}
|
|
|
|
def sample():
|
|
while not stop.is_set():
|
|
if containers:
|
|
result = docker(
|
|
'stats', '--no-stream', '--format',
|
|
'{{.Name}} {{.MemUsage}}', *containers,
|
|
check_result=False,
|
|
)
|
|
for line in result.stdout.splitlines():
|
|
parts = line.split()
|
|
if len(parts) < 2:
|
|
continue
|
|
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
|
|
if not match:
|
|
continue
|
|
value = float(match.group(1))
|
|
if match.group(2) == 'GiB':
|
|
value *= 1024
|
|
elif match.group(2) == 'KiB':
|
|
value /= 1024
|
|
peak['mb'] = max(peak['mb'], value)
|
|
stop.wait(0.3)
|
|
|
|
sampler = threading.Thread(target=sample, daemon=True)
|
|
sampler.start()
|
|
try:
|
|
def do_upload(index):
|
|
client = new_client(ctx['base_url'])
|
|
return upload(client, ctx['alice'], ctx['chat'], data,
|
|
'big-' + str(index) + '.png')
|
|
|
|
with ThreadPoolExecutor(max_workers=2) as pool:
|
|
attachments = list(pool.map(do_upload, (1, 2)))
|
|
ctx['attachment_id'] = attachments[0]['id']
|
|
for attachment in attachments:
|
|
ctx['tracker'].track(attachment['id'])
|
|
results = [wait_ready(ctx['client'], ctx['alice'],
|
|
attachment['id'], 120, allow_failed=True)
|
|
for attachment in attachments]
|
|
finally:
|
|
stop.set()
|
|
sampler.join(timeout=5)
|
|
for meta in results:
|
|
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
|
|
if meta['status'] == 'failed':
|
|
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
|
|
'unexpected big-image error code')
|
|
for container in containers:
|
|
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
|
|
check_result=False)
|
|
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
|
|
return {'attachment_id': attachments[0]['id'],
|
|
'job_attempts': [],
|
|
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
|
|
|
|
|
|
SCENARIOS = {
|
|
'duplicate': scenario_duplicate,
|
|
'reorder': scenario_reorder,
|
|
'crash_before_side_effect': scenario_crash_before_side_effect,
|
|
'crash_after_side_effect': scenario_crash_after_side_effect,
|
|
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
|
|
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
|
|
'relay_crash_after_publish': scenario_relay_crash_after_publish,
|
|
'stale_version': scenario_stale_version,
|
|
'poison': scenario_poison,
|
|
'retry_and_replay': scenario_retry_and_replay,
|
|
'hung_worker': scenario_hung_worker,
|
|
'big_image': scenario_big_image,
|
|
}
|
|
|
|
|
|
def select(name):
|
|
if name == 'all':
|
|
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
|
|
if name == 'level4':
|
|
return LEVEL4
|
|
if name == 'track2':
|
|
return TRACK2
|
|
if name in SCENARIOS:
|
|
return [name]
|
|
raise SystemExit('unknown scenario: ' + name)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default=None)
|
|
parser.add_argument('--ca-file', default=None)
|
|
parser.add_argument('--scenario', default='level4')
|
|
parser.add_argument('--repeat', type=int, default=1)
|
|
parser.add_argument('--output', default=None)
|
|
args = parser.parse_args()
|
|
|
|
if worker_env('FAULT_INJECTION_ENABLED') != '1':
|
|
print('fault injection is disabled; restart the stand with:\n ' + HINT,
|
|
file=sys.stderr)
|
|
raise SystemExit(2)
|
|
|
|
client = new_client(args.base_url, args.ca_file)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
clear_faults()
|
|
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
|
|
ready = wait_ready(client, alice, reference['id'], 60)
|
|
baseline = variant_sha(ready)
|
|
|
|
selected = select(args.scenario)
|
|
results = []
|
|
failed = 0
|
|
for name in selected:
|
|
for repeat in range(1, args.repeat + 1):
|
|
clear_faults()
|
|
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
|
|
args.ca_file)
|
|
tracker.start()
|
|
started = time.monotonic()
|
|
ctx = {
|
|
'client': client,
|
|
'base_url': args.base_url or 'http://localhost:8080',
|
|
'alice': alice,
|
|
'chat': chat,
|
|
'tracker': tracker,
|
|
'baseline': baseline,
|
|
'reference_id': reference['id'],
|
|
'ca_file': args.ca_file,
|
|
}
|
|
scenario = {'attachment_id': None, 'job_attempts': [],
|
|
'notes': ''}
|
|
passed = True
|
|
try:
|
|
scenario.update(SCENARIOS[name](ctx))
|
|
except Exception as error: # noqa: BLE001 - scenario boundary
|
|
passed = False
|
|
scenario['notes'] = type(error).__name__ + ': ' + str(error)
|
|
finally:
|
|
tracker.stop()
|
|
clear_faults()
|
|
compose('start', 'worker', check_result=False)
|
|
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
|
|
try:
|
|
invariants = evaluate(
|
|
tracker, attachment_id, baseline,
|
|
compare_baseline=ctx.get('compare_baseline', True),
|
|
)
|
|
except Exception: # noqa: BLE001
|
|
invariants = {key: False for key in
|
|
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
|
|
passed = passed and all(invariants.values())
|
|
if not passed:
|
|
failed += 1
|
|
result = {
|
|
'name': name,
|
|
'repeat': repeat,
|
|
'passed': passed,
|
|
'duration_s': round(time.monotonic() - started, 3),
|
|
'attachment_id': attachment_id,
|
|
'job_attempts': scenario['job_attempts'],
|
|
'events_summary': scenario.get('events_summary', {}),
|
|
'invariants': invariants,
|
|
'notes': scenario['notes'],
|
|
}
|
|
results.append(result)
|
|
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
|
|
' ' + str(result['duration_s']) + 's ' +
|
|
json.dumps(invariants) + ' ' + result['notes'],
|
|
flush=True)
|
|
|
|
if args.output:
|
|
path = Path(args.output)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
|
|
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
|
|
raise SystemExit(1 if failed else 0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|