forked from mtusi2026/mtusi2026lab6
lab6: базовый коммит из lab5 (0790599)
Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.
Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.
Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
(OTEL_*) из skeleton.
This commit is contained in:
Executable
+749
@@ -0,0 +1,749 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
|
||||
|
||||
from stand import ( # noqa: E402
|
||||
ROOT,
|
||||
Failure,
|
||||
check,
|
||||
clear_fault,
|
||||
clear_faults,
|
||||
cli,
|
||||
compose,
|
||||
docker,
|
||||
new_client,
|
||||
rabbit_queues,
|
||||
sample_bytes,
|
||||
set_fault,
|
||||
sha256,
|
||||
upload,
|
||||
variant_sha,
|
||||
wait_for,
|
||||
wait_ready,
|
||||
worker_env,
|
||||
worker_start_count,
|
||||
)
|
||||
from pngkit import worst_decode_png # noqa: E402
|
||||
|
||||
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
|
||||
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
|
||||
'make stand-up')
|
||||
|
||||
LEVEL4 = [
|
||||
'crash_after_commit_before_publish',
|
||||
'crash_after_side_effect',
|
||||
'crash_after_commit_before_ack',
|
||||
'retry_and_replay',
|
||||
'hung_worker',
|
||||
'poison',
|
||||
'big_image',
|
||||
]
|
||||
TRACK2 = [
|
||||
'duplicate',
|
||||
'reorder',
|
||||
'crash_before_side_effect',
|
||||
'crash_after_side_effect',
|
||||
'crash_after_commit_before_ack',
|
||||
'relay_crash_after_publish',
|
||||
'stale_version',
|
||||
]
|
||||
|
||||
|
||||
class Tracker(threading.Thread):
|
||||
"""Polls metadata while a scenario runs to prove readiness never regresses."""
|
||||
|
||||
def __init__(self, base_url, actor, ca_file=None):
|
||||
super().__init__(daemon=True)
|
||||
self.client = new_client(base_url, ca_file)
|
||||
self.actor = actor
|
||||
self.ids = []
|
||||
self.records = {}
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
def track(self, attachment_id):
|
||||
if attachment_id not in self.records:
|
||||
self.ids.append(attachment_id)
|
||||
self.records[attachment_id] = []
|
||||
|
||||
def run(self):
|
||||
while not self.stop_event.is_set():
|
||||
for attachment_id in list(self.ids):
|
||||
try:
|
||||
meta = self.client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment_id,
|
||||
actor=self.actor, schema='Attachment',
|
||||
)['json']
|
||||
self.records[attachment_id].append(
|
||||
(time.monotonic(), meta['status'],
|
||||
meta.get('ready_pipeline_version'))
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self.stop_event.wait(0.2)
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
self.join(timeout=5)
|
||||
|
||||
|
||||
def build_message(status, job, attempt=None):
|
||||
created = next(
|
||||
event for event in status['events']
|
||||
if event['job_id'] == job['job_id'] and event['event'] == 'created'
|
||||
)
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': job['job_id'],
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': status['attachment']['id'],
|
||||
'pipeline_version': job['pipeline_version'],
|
||||
'attempt': attempt if attempt is not None else job['attempt'],
|
||||
'created_at': created['created_at'],
|
||||
}
|
||||
|
||||
|
||||
def publish_raw(messages):
|
||||
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
|
||||
for message in messages)
|
||||
code, payload = cli('publish-raw', stdin=text)
|
||||
check(code == 0, 'publish-raw failed: ' + str(payload))
|
||||
return payload
|
||||
|
||||
|
||||
def status_of(attachment_id):
|
||||
code, payload = cli('status', attachment_id)
|
||||
check(code == 0, 'status failed: ' + str(payload))
|
||||
return payload
|
||||
|
||||
|
||||
def flush_outbox(attachment_id):
|
||||
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
||||
30, 'outbox publish', 1.0)
|
||||
|
||||
|
||||
def new_attachment_id(chat_id, before):
|
||||
rows = cli('find-by-chat', chat_id)[1]
|
||||
for row in reversed(rows):
|
||||
if row['id'] not in before:
|
||||
return row['id']
|
||||
return None
|
||||
|
||||
|
||||
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
|
||||
compare_baseline=True):
|
||||
if attachment_id is None:
|
||||
return {'I1': True, 'I2': True, 'I3': True,
|
||||
'I4': True, 'I5': True, 'I6': True}
|
||||
status = status_of(attachment_id)
|
||||
attachment = status['attachment']
|
||||
jobs = status['jobs']
|
||||
variants = status['variants']
|
||||
ready_version = attachment['ready_pipeline_version']
|
||||
|
||||
records = tracker.records.get(attachment_id, [])
|
||||
reached = any(record[1] == ('failed' if expect_failed else 'ready')
|
||||
for record in records)
|
||||
if not records:
|
||||
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
|
||||
i1 = reached or attachment['status'] in ('ready', 'failed')
|
||||
|
||||
ready_variants = [v for v in variants
|
||||
if v['pipeline_version'] == ready_version]
|
||||
succeeded_ids = {job['job_id'] for job in jobs
|
||||
if job['status'] == 'succeeded'}
|
||||
succeeded_events = {}
|
||||
for event in status['events']:
|
||||
if event['event'] == 'succeeded':
|
||||
succeeded_events[event['job_id']] = (
|
||||
succeeded_events.get(event['job_id'], 0) + 1
|
||||
)
|
||||
i2 = (len(ready_variants) == 3
|
||||
and all(succeeded_events.get(job_id) == 1
|
||||
for job_id in succeeded_ids))
|
||||
|
||||
i3 = True
|
||||
seen_ready = False
|
||||
last_version = 0
|
||||
for _at, state, version in records:
|
||||
if seen_ready and state != 'ready':
|
||||
i3 = False
|
||||
if state == 'ready':
|
||||
seen_ready = True
|
||||
if version is not None:
|
||||
if version < last_version:
|
||||
i3 = False
|
||||
last_version = version
|
||||
|
||||
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
|
||||
keys = {(variant['pipeline_version'], variant['variant'])
|
||||
for variant in variants}
|
||||
i5 = len(keys) == len(variants)
|
||||
|
||||
if compare_baseline and ready_version == 1 and ready_variants:
|
||||
observed = {variant['variant']: variant['sha256']
|
||||
for variant in ready_variants}
|
||||
i6 = all(observed.get(name) == digest
|
||||
for name, digest in baseline.items())
|
||||
else:
|
||||
i6 = True
|
||||
|
||||
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
|
||||
'I5': i5, 'I6': i6}
|
||||
|
||||
|
||||
# --- scenarios -----------------------------------------------------------
|
||||
|
||||
|
||||
def scenario_duplicate(ctx):
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'duplicate.png')
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
||||
status = status_of(attachment['id'])
|
||||
before = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']}
|
||||
message = build_message(status, status['jobs'][0])
|
||||
publish_raw([message] * 5)
|
||||
wait_for(
|
||||
lambda: len([event for event in status_of(attachment['id'])['events']
|
||||
if event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'already_succeeded']) == 5,
|
||||
20, 'five duplicate_skipped events', 1.0,
|
||||
)
|
||||
after_status = status_of(attachment['id'])
|
||||
after = {variant['variant']: variant['s3']
|
||||
for variant in after_status['variants']}
|
||||
check(before == after, 'duplicate delivery rewrote variant objects')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
||||
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
|
||||
|
||||
|
||||
def scenario_reorder(ctx):
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'reorder.png')
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
flush_outbox(attachment['id'])
|
||||
status = status_of(attachment['id'])
|
||||
message = build_message(status, status['jobs'][0])
|
||||
stale = dict(message, attempt=99)
|
||||
publish_raw([stale, message])
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
after_status = status_of(attachment['id'])
|
||||
skipped = [event for event in after_status['events']
|
||||
if event['event'] == 'duplicate_skipped']
|
||||
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
|
||||
check(len([event for event in after_status['events']
|
||||
if event['event'] == 'succeeded']) == 1,
|
||||
'expected exactly one successful effect')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
||||
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
|
||||
|
||||
|
||||
def crash_scenario(ctx, point, filename, expected_attempt,
|
||||
expect_lease_expired=False, expect_duplicate=False):
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), filename)
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
starts = worker_start_count()
|
||||
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment['id'])
|
||||
job = status['jobs'][0]
|
||||
check(job['attempt'] == expected_attempt,
|
||||
'expected attempt ' + str(expected_attempt) +
|
||||
', got ' + str(job['attempt']))
|
||||
check(worker_start_count() >= starts + 2,
|
||||
'worker process did not restart after the injected crash')
|
||||
if expect_lease_expired:
|
||||
check(any(event['event'] == 'lease_expired'
|
||||
for event in status['events']),
|
||||
'missing lease_expired event')
|
||||
if expect_duplicate:
|
||||
check(any(event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'already_succeeded'
|
||||
for event in status['events']),
|
||||
'missing already_succeeded duplicate after redelivery')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt']],
|
||||
'notes': 'crashed at ' + point}
|
||||
|
||||
|
||||
def scenario_crash_before_side_effect(ctx):
|
||||
return crash_scenario(ctx, 'worker.before_side_effect',
|
||||
'crash-before.png', 2, expect_lease_expired=True)
|
||||
|
||||
|
||||
def scenario_crash_after_side_effect(ctx):
|
||||
return crash_scenario(ctx, 'worker.after_side_effect',
|
||||
'crash-after.png', 2, expect_lease_expired=True)
|
||||
|
||||
|
||||
def scenario_crash_after_commit_before_ack(ctx):
|
||||
return crash_scenario(ctx, 'worker.after_commit_before_ack',
|
||||
'crash-ack.png', 1, expect_duplicate=True)
|
||||
|
||||
|
||||
def scenario_crash_after_commit_before_publish(ctx):
|
||||
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
||||
set_fault('fault:api.after_commit_before_publish', 'crash')
|
||||
try:
|
||||
try:
|
||||
ctx['client'].call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + ctx['chat']['id'] +
|
||||
'/attachments?filename=crash-publish.png',
|
||||
(202, 502, 503, 504),
|
||||
actor=ctx['alice'],
|
||||
raw=sample_bytes(),
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
except Failure:
|
||||
# A killed replica may close the connection instead of answering.
|
||||
pass
|
||||
finally:
|
||||
clear_fault('fault:api.after_commit_before_publish')
|
||||
attachment_id = wait_for(
|
||||
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
||||
'attachment row after API crash', 1.0,
|
||||
)
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
status = status_of(attachment_id)
|
||||
check(len([event for event in status['events']
|
||||
if event['event'] == 'created']) == 1,
|
||||
'expected exactly one created event')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'outbox relay published the committed job after API restart'}
|
||||
|
||||
|
||||
def scenario_relay_crash_after_publish(ctx):
|
||||
compose('stop', 'worker')
|
||||
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
||||
try:
|
||||
set_fault('fault:relay.after_publish_before_mark', 'crash')
|
||||
try:
|
||||
ctx['client'].call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + ctx['chat']['id'] +
|
||||
'/attachments?filename=relay-crash.png',
|
||||
(202, 502, 503, 504),
|
||||
actor=ctx['alice'],
|
||||
raw=sample_bytes(),
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
except Failure:
|
||||
# The relay may crash the handling replica before the reply flushes.
|
||||
pass
|
||||
clear_fault('fault:relay.after_publish_before_mark')
|
||||
attachment_id = wait_for(
|
||||
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
||||
'attachment row after relay crash', 1.0,
|
||||
)
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
||||
60, 'outbox published after relay crash', 1.0)
|
||||
queue_depth = rabbit_queues().get('attachments.process', {}).get(
|
||||
'messages', 0)
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
finally:
|
||||
clear_fault('fault:relay.after_publish_before_mark')
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment_id)
|
||||
check(len([event for event in status['events']
|
||||
if event['event'] == 'succeeded']) == 1,
|
||||
'expected exactly one succeeded event')
|
||||
duplicated = any(event['event'] == 'duplicate_skipped'
|
||||
for event in status['events'])
|
||||
note = ('relay republished the same message; duplicate absorbed'
|
||||
if duplicated else
|
||||
'relay published once in this run; I1/I2 still hold')
|
||||
if queue_depth >= 2:
|
||||
check(duplicated, 'two queued copies but no duplicate_skipped event')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': note}
|
||||
|
||||
|
||||
def scenario_stale_version(ctx):
|
||||
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
|
||||
'stale_version needs WORKER_CONCURRENCY>=2')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'stale.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
flush_outbox(attachment_id)
|
||||
code, payload = cli('reprocess', attachment_id,
|
||||
'--pipeline-version', '2', '--reason', 'suite')
|
||||
check(code == 0, 'reprocess v2 failed: ' + str(payload))
|
||||
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
||||
'delay:8000')
|
||||
compose('start', 'worker')
|
||||
v2_etags = None
|
||||
|
||||
def both_terminal():
|
||||
nonlocal v2_etags
|
||||
status = status_of(attachment_id)
|
||||
jobs = {(job['pipeline_version']): job for job in status['jobs']}
|
||||
if '2' in jobs or 2 in jobs:
|
||||
v2_job = jobs.get(2)
|
||||
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
|
||||
v2_etags = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']
|
||||
if variant['pipeline_version'] == 2}
|
||||
v1_job = jobs.get(1)
|
||||
v2_job = jobs.get(2)
|
||||
return (v1_job and v2_job
|
||||
and v1_job['status'] == 'superseded'
|
||||
and v2_job['status'] == 'succeeded')
|
||||
|
||||
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
|
||||
status = status_of(attachment_id)
|
||||
check(status['attachment']['ready_pipeline_version'] == 2,
|
||||
'ready pipeline version is not 2')
|
||||
check(all(variant['pipeline_version'] == 2
|
||||
for variant in status['variants']),
|
||||
'variant rows from v1 survived')
|
||||
after_etags = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']
|
||||
if variant['pipeline_version'] == 2}
|
||||
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
|
||||
|
||||
served = ctx['client'].call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id +
|
||||
'/content?variant=thumbnail',
|
||||
actor=ctx['alice'],
|
||||
)
|
||||
v2_thumbnail = next(variant for variant in status['variants']
|
||||
if variant['variant'] == 'thumbnail')
|
||||
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
|
||||
'served thumbnail is not the v2 object')
|
||||
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
|
||||
'served thumbnail equals the v1 baseline')
|
||||
|
||||
v1_job = next(job for job in status['jobs']
|
||||
if job['pipeline_version'] == 1)
|
||||
publish_raw([build_message(status, v1_job)])
|
||||
wait_for(
|
||||
lambda: any(event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'superseded'
|
||||
for event in status_of(attachment_id)['events']),
|
||||
20, 'superseded duplicate skip', 1.0,
|
||||
)
|
||||
finally:
|
||||
clear_fault('fault:worker.after_side_effect:' +
|
||||
(attachment['id'] if 'attachment' in locals() else ''))
|
||||
compose('start', 'worker')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'v1 finished after v2 and was recorded as superseded'}
|
||||
|
||||
|
||||
def scenario_poison(ctx):
|
||||
before = rabbit_queues().get('attachments.process.dlq', {}).get(
|
||||
'messages', 0)
|
||||
publish_raw(['not-json', '{"schema_version":1}'])
|
||||
wait_for(
|
||||
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
|
||||
'messages', 0) >= before + 2,
|
||||
15, 'two poison messages in DLQ', 0.5,
|
||||
)
|
||||
return {'attachment_id': None, 'job_attempts': [],
|
||||
'notes': 'malformed messages moved to the DLQ without touching jobs'}
|
||||
|
||||
|
||||
def scenario_retry_and_replay(ctx):
|
||||
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
|
||||
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
|
||||
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'retry.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
|
||||
'transient')
|
||||
compose('start', 'worker')
|
||||
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
|
||||
90, allow_failed=True)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
check(failed['status'] == 'failed', 'transient fault was not exhausted')
|
||||
check(failed['error_code'] == 'retry_exhausted',
|
||||
'expected retry_exhausted, got ' + str(failed['error_code']))
|
||||
status = status_of(attachment_id)
|
||||
retries = [event for event in status['events']
|
||||
if event['event'] == 'retry_scheduled']
|
||||
check(len(retries) == max_attempts - 1,
|
||||
'expected ' + str(max_attempts - 1) + ' retries, got ' +
|
||||
str(len(retries)))
|
||||
for index, event in enumerate(retries):
|
||||
limit = min(cap, base * 2 ** index)
|
||||
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
|
||||
'retry delay exceeds its jitter cap')
|
||||
ctx['client'].call(
|
||||
'GET', '/api/v1/attachments/' + attachment_id + '/content',
|
||||
409, actor=ctx['alice'],
|
||||
)
|
||||
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
|
||||
check(code == 3 and payload.get('reason') in ('no_failed_job',
|
||||
'already_ready'),
|
||||
'replay of a ready attachment was not refused')
|
||||
|
||||
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
|
||||
code, payload = cli('replay', attachment_id, '--reason', 'suite')
|
||||
check(code == 0, 'operator replay failed: ' + str(payload))
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
|
||||
status = status_of(attachment_id)
|
||||
replayed = [event for event in status['events']
|
||||
if event['event'] == 'replayed']
|
||||
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
|
||||
'missing replayed event or operator audit')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'transient injected fault exhausted retries; replay recovered'}
|
||||
|
||||
|
||||
def scenario_hung_worker(ctx):
|
||||
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'hung.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
||||
'delay:' + str((timeout_seconds + 5) * 1000))
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment_id)
|
||||
check(any(event['event'] == 'retry_scheduled'
|
||||
and event['detail'].get('code') == 'processing_timeout'
|
||||
for event in status['events']),
|
||||
'missing processing_timeout retry')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'hard timeout aborted the hung render and retried'}
|
||||
|
||||
|
||||
def scenario_big_image(ctx):
|
||||
ctx['compare_baseline'] = False
|
||||
data = worst_decode_png()
|
||||
containers = compose('ps', '-q', 'worker').stdout.split()
|
||||
stop = threading.Event()
|
||||
peak = {'mb': 0.0}
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
if containers:
|
||||
result = docker(
|
||||
'stats', '--no-stream', '--format',
|
||||
'{{.Name}} {{.MemUsage}}', *containers,
|
||||
check_result=False,
|
||||
)
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
|
||||
if not match:
|
||||
continue
|
||||
value = float(match.group(1))
|
||||
if match.group(2) == 'GiB':
|
||||
value *= 1024
|
||||
elif match.group(2) == 'KiB':
|
||||
value /= 1024
|
||||
peak['mb'] = max(peak['mb'], value)
|
||||
stop.wait(0.3)
|
||||
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
sampler.start()
|
||||
try:
|
||||
def do_upload(index):
|
||||
client = new_client(ctx['base_url'])
|
||||
return upload(client, ctx['alice'], ctx['chat'], data,
|
||||
'big-' + str(index) + '.png')
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
attachments = list(pool.map(do_upload, (1, 2)))
|
||||
ctx['attachment_id'] = attachments[0]['id']
|
||||
for attachment in attachments:
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
results = [wait_ready(ctx['client'], ctx['alice'],
|
||||
attachment['id'], 120, allow_failed=True)
|
||||
for attachment in attachments]
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join(timeout=5)
|
||||
for meta in results:
|
||||
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
|
||||
if meta['status'] == 'failed':
|
||||
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
|
||||
'unexpected big-image error code')
|
||||
for container in containers:
|
||||
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
|
||||
check_result=False)
|
||||
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
|
||||
return {'attachment_id': attachments[0]['id'],
|
||||
'job_attempts': [],
|
||||
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'duplicate': scenario_duplicate,
|
||||
'reorder': scenario_reorder,
|
||||
'crash_before_side_effect': scenario_crash_before_side_effect,
|
||||
'crash_after_side_effect': scenario_crash_after_side_effect,
|
||||
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
|
||||
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
|
||||
'relay_crash_after_publish': scenario_relay_crash_after_publish,
|
||||
'stale_version': scenario_stale_version,
|
||||
'poison': scenario_poison,
|
||||
'retry_and_replay': scenario_retry_and_replay,
|
||||
'hung_worker': scenario_hung_worker,
|
||||
'big_image': scenario_big_image,
|
||||
}
|
||||
|
||||
|
||||
def select(name):
|
||||
if name == 'all':
|
||||
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
|
||||
if name == 'level4':
|
||||
return LEVEL4
|
||||
if name == 'track2':
|
||||
return TRACK2
|
||||
if name in SCENARIOS:
|
||||
return [name]
|
||||
raise SystemExit('unknown scenario: ' + name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=None)
|
||||
parser.add_argument('--ca-file', default=None)
|
||||
parser.add_argument('--scenario', default='level4')
|
||||
parser.add_argument('--repeat', type=int, default=1)
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
if worker_env('FAULT_INJECTION_ENABLED') != '1':
|
||||
print('fault injection is disabled; restart the stand with:\n ' + HINT,
|
||||
file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
client = new_client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
clear_faults()
|
||||
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
|
||||
ready = wait_ready(client, alice, reference['id'], 60)
|
||||
baseline = variant_sha(ready)
|
||||
|
||||
selected = select(args.scenario)
|
||||
results = []
|
||||
failed = 0
|
||||
for name in selected:
|
||||
for repeat in range(1, args.repeat + 1):
|
||||
clear_faults()
|
||||
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
|
||||
args.ca_file)
|
||||
tracker.start()
|
||||
started = time.monotonic()
|
||||
ctx = {
|
||||
'client': client,
|
||||
'base_url': args.base_url or 'http://localhost:8080',
|
||||
'alice': alice,
|
||||
'chat': chat,
|
||||
'tracker': tracker,
|
||||
'baseline': baseline,
|
||||
'reference_id': reference['id'],
|
||||
'ca_file': args.ca_file,
|
||||
}
|
||||
scenario = {'attachment_id': None, 'job_attempts': [],
|
||||
'notes': ''}
|
||||
passed = True
|
||||
try:
|
||||
scenario.update(SCENARIOS[name](ctx))
|
||||
except Exception as error: # noqa: BLE001 - scenario boundary
|
||||
passed = False
|
||||
scenario['notes'] = type(error).__name__ + ': ' + str(error)
|
||||
finally:
|
||||
tracker.stop()
|
||||
clear_faults()
|
||||
compose('start', 'worker', check_result=False)
|
||||
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
|
||||
try:
|
||||
invariants = evaluate(
|
||||
tracker, attachment_id, baseline,
|
||||
compare_baseline=ctx.get('compare_baseline', True),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
invariants = {key: False for key in
|
||||
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
|
||||
passed = passed and all(invariants.values())
|
||||
if not passed:
|
||||
failed += 1
|
||||
result = {
|
||||
'name': name,
|
||||
'repeat': repeat,
|
||||
'passed': passed,
|
||||
'duration_s': round(time.monotonic() - started, 3),
|
||||
'attachment_id': attachment_id,
|
||||
'job_attempts': scenario['job_attempts'],
|
||||
'events_summary': scenario.get('events_summary', {}),
|
||||
'invariants': invariants,
|
||||
'notes': scenario['notes'],
|
||||
}
|
||||
results.append(result)
|
||||
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
|
||||
' ' + str(result['duration_s']) + 's ' +
|
||||
json.dumps(invariants) + ' ' + result['notes'],
|
||||
flush=True)
|
||||
|
||||
if args.output:
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
|
||||
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
|
||||
raise SystemExit(1 if failed else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Shared stdlib-only helpers for the lab5 demo and fault suite."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
ENV_FILE = '.env'
|
||||
COMPOSE = ['docker', 'compose', '--env-file', ENV_FILE]
|
||||
|
||||
|
||||
def compose(*args, stdin=None, check_result=True):
|
||||
result = subprocess.run(
|
||||
COMPOSE + list(args),
|
||||
cwd=ROOT,
|
||||
input=stdin,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
if check_result and result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
raise Failure('compose ' + ' '.join(args) + ' failed: ' +
|
||||
(detail[-1] if detail else 'unknown error'))
|
||||
return result
|
||||
|
||||
|
||||
def docker(*args, check_result=True):
|
||||
result = subprocess.run(
|
||||
['docker'] + list(args), cwd=ROOT, text=True, capture_output=True,
|
||||
)
|
||||
if check_result and result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
raise Failure('docker ' + ' '.join(args) + ' failed: ' +
|
||||
(detail[-1] if detail else 'unknown error'))
|
||||
return result
|
||||
|
||||
|
||||
def cli(*args, stdin=None):
|
||||
"""Run jobs.cli in the API container; returns (exit_code, parsed_json)."""
|
||||
for _ in range(5):
|
||||
result = compose(
|
||||
'exec', '-T', 'api', 'node', 'dist/jobs/jobs.cli.js', *args,
|
||||
stdin=stdin, check_result=False,
|
||||
)
|
||||
text = result.stdout.strip()
|
||||
if text:
|
||||
try:
|
||||
return result.returncode, json.loads(text)
|
||||
except ValueError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise Failure('jobs.cli did not return JSON for ' + ' '.join(args))
|
||||
|
||||
|
||||
def set_fault(key, value):
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'SET', key, value)
|
||||
|
||||
|
||||
def clear_fault(key):
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key)
|
||||
|
||||
|
||||
def clear_faults():
|
||||
script = (
|
||||
"local keys=redis.call('keys',ARGV[1]) "
|
||||
"for i=1,#keys do redis.call('del',keys[i]) end return #keys"
|
||||
)
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', script, '0', 'fault:*')
|
||||
|
||||
|
||||
def rabbit_queues():
|
||||
result = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q',
|
||||
'-p', 'messenger', 'list_queues',
|
||||
'name', 'messages', 'type')
|
||||
queues = {}
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and parts[0] not in ('name', 'Timeout:'):
|
||||
try:
|
||||
queues[parts[0]] = {'messages': int(parts[1]),
|
||||
'type': parts[2] if len(parts) > 2 else ''}
|
||||
except ValueError:
|
||||
continue
|
||||
return queues
|
||||
|
||||
|
||||
def wait_for(predicate, timeout, description='condition', interval=0.5):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = predicate()
|
||||
if last:
|
||||
return last
|
||||
time.sleep(interval)
|
||||
raise Failure('timed out waiting for ' + description)
|
||||
|
||||
|
||||
def upload(client, actor, chat, data, filename='sample.png',
|
||||
content_type='image/png'):
|
||||
result = client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
|
||||
202,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={'Content-Type': content_type},
|
||||
schema='Attachment',
|
||||
)
|
||||
check(result['json']['status'] == 'queued', 'upload did not answer queued')
|
||||
return result['json']
|
||||
|
||||
|
||||
def metadata(client, actor, attachment_id):
|
||||
return client.call('GET', '/api/v1/attachments/' + attachment_id,
|
||||
actor=actor, schema='Attachment')['json']
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment_id, timeout=60,
|
||||
allow_failed=False):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
meta = metadata(client, actor, attachment_id)
|
||||
if meta['status'] == 'failed':
|
||||
if allow_failed:
|
||||
return meta
|
||||
raise Failure('attachment failed: ' +
|
||||
str(meta.get('error_code')))
|
||||
if meta['status'] == 'ready':
|
||||
return meta
|
||||
time.sleep(0.2)
|
||||
raise Failure('attachment did not become ready in ' + str(timeout) + 's')
|
||||
|
||||
|
||||
def sha256(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def variant_sha(meta):
|
||||
return {item['name']: item['sha256'] for item in meta['variants']}
|
||||
|
||||
|
||||
def sample_bytes():
|
||||
return (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
|
||||
def worker_env(name):
|
||||
result = compose('exec', '-T', 'worker', 'printenv', name,
|
||||
check_result=False)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def worker_start_count():
|
||||
result = compose('logs', '--no-color', 'worker', check_result=False)
|
||||
return result.stdout.count('worker ready:')
|
||||
|
||||
|
||||
def worker_memory_peak(containers):
|
||||
names = [line for line in containers.splitlines() if line.strip()]
|
||||
if not names:
|
||||
return 'n/a'
|
||||
result = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format',
|
||||
'{{.Name}} {{.MemUsage}}'] + names,
|
||||
text=True, capture_output=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def print_step(number, text):
|
||||
print('[' + str(number) + '] ' + text, flush=True)
|
||||
|
||||
|
||||
def new_client(base_url, ca_file=None):
|
||||
return Client(base_url, ca_file)
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level-3 demo: work accepted while the worker is down survives restarts."""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from stand import ( # noqa: E402
|
||||
Failure,
|
||||
check,
|
||||
cli,
|
||||
compose,
|
||||
new_client,
|
||||
print_step,
|
||||
rabbit_queues,
|
||||
sample_bytes,
|
||||
sha256,
|
||||
upload,
|
||||
wait_for,
|
||||
wait_ready,
|
||||
)
|
||||
from client import arguments, run # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('lab5 worker-down durability demo')
|
||||
client = new_client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
data = sample_bytes()
|
||||
|
||||
print_step(1, 'compose ps')
|
||||
print(compose('ps', '--format',
|
||||
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
||||
|
||||
print_step(2, 'stop worker')
|
||||
compose('stop', 'worker')
|
||||
|
||||
print_step(3, 'upload while worker is down (expect 202 queued)')
|
||||
attachment = upload(client, alice, chat, data, 'durability.png')
|
||||
attachment_id = attachment['id']
|
||||
print('attachment=' + attachment_id + ' status=' + attachment['status'])
|
||||
|
||||
print_step(4, 'content and thumbnail must answer 409 before ready')
|
||||
for query in ('', '?variant=thumbnail'):
|
||||
result = client.call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id + '/content' + query,
|
||||
409,
|
||||
actor=alice,
|
||||
)
|
||||
check(result['status'] == 409, 'content was served before ready')
|
||||
|
||||
print_step(5, 'link the queued attachment to messages')
|
||||
with_attachment = client.message(
|
||||
alice, chat, '', attachments=[attachment_id],
|
||||
)['json']
|
||||
plain = client.message(alice, chat, 'plain while worker is down')['json']
|
||||
check(with_attachment['attachment_ids'] == [attachment_id],
|
||||
'message lost the queued attachment')
|
||||
check(plain['attachment_ids'] == [], 'plain message gained an attachment')
|
||||
print('message=' + with_attachment['id'])
|
||||
|
||||
print_step(6, 'wait for outbox publish and a message in the queue')
|
||||
def published():
|
||||
_code, status = cli('status', attachment_id)
|
||||
queues = rabbit_queues()
|
||||
return (
|
||||
status['outbox']['pending'] == 0
|
||||
and queues.get('attachments.process', {}).get('messages', 0) >= 1
|
||||
)
|
||||
wait_for(published, 30, 'outbox publish and queue depth >= 1')
|
||||
queues = rabbit_queues()
|
||||
print('queue=' + str(queues.get('attachments.process')))
|
||||
|
||||
print_step(7, 'restart broker and API with volumes')
|
||||
compose('restart', 'broker', 'api')
|
||||
compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api')
|
||||
|
||||
print_step(8, 'the quorum message survived the broker restart')
|
||||
def survived():
|
||||
depth = rabbit_queues().get('attachments.process', {}).get('messages', 0)
|
||||
return depth >= 1
|
||||
wait_for(survived, 60, 'queue depth >= 1 after broker restart')
|
||||
print('queue=' + str(rabbit_queues().get('attachments.process')))
|
||||
|
||||
print_step(9, 'start worker')
|
||||
compose('start', 'worker')
|
||||
|
||||
print_step(10, 'wait ready and verify the thumbnail')
|
||||
ready = wait_ready(client, alice, attachment_id, 60)
|
||||
thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
||||
content = client.call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail',
|
||||
actor=alice,
|
||||
)
|
||||
check(jpeg_size(content['bytes']) == (256, 160),
|
||||
'thumbnail is not 256x160 JPEG')
|
||||
check(len(content['bytes']) == thumbnail['size_bytes'],
|
||||
'thumbnail size differs from metadata')
|
||||
check(sha256(content['bytes']) == thumbnail['sha256'],
|
||||
'thumbnail checksum differs from metadata')
|
||||
history = client.call(
|
||||
'GET',
|
||||
'/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
||||
actor=alice,
|
||||
schema='MessagePage',
|
||||
)['json']
|
||||
check(
|
||||
any(attachment_id in item['attachment_ids']
|
||||
for item in history['items']),
|
||||
'message history lost the attachment',
|
||||
)
|
||||
print('ready attachment=' + attachment_id +
|
||||
' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...')
|
||||
|
||||
print_step(11, 'compose ps')
|
||||
print(compose('ps', '--format',
|
||||
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
||||
print('PASS: worker-down durability demo')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user