lab5: e2e, fault suite и доказательства

This commit is contained in:
2026-11-05 01:17:58 +03:00
parent d19824f546
commit 551cf70989
28 changed files with 2269 additions and 60 deletions
+749
View File
@@ -0,0 +1,749 @@
#!/usr/bin/env python3
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
import argparse
import json
import re
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
from stand import ( # noqa: E402
ROOT,
Failure,
check,
clear_fault,
clear_faults,
cli,
compose,
docker,
new_client,
rabbit_queues,
sample_bytes,
set_fault,
sha256,
upload,
variant_sha,
wait_for,
wait_ready,
worker_env,
worker_start_count,
)
from pngkit import worst_decode_png # noqa: E402
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
'make stand-up')
LEVEL4 = [
'crash_after_commit_before_publish',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'retry_and_replay',
'hung_worker',
'poison',
'big_image',
]
TRACK2 = [
'duplicate',
'reorder',
'crash_before_side_effect',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'relay_crash_after_publish',
'stale_version',
]
class Tracker(threading.Thread):
"""Polls metadata while a scenario runs to prove readiness never regresses."""
def __init__(self, base_url, actor, ca_file=None):
super().__init__(daemon=True)
self.client = new_client(base_url, ca_file)
self.actor = actor
self.ids = []
self.records = {}
self.stop_event = threading.Event()
def track(self, attachment_id):
if attachment_id not in self.records:
self.ids.append(attachment_id)
self.records[attachment_id] = []
def run(self):
while not self.stop_event.is_set():
for attachment_id in list(self.ids):
try:
meta = self.client.call(
'GET', '/api/v1/attachments/' + attachment_id,
actor=self.actor, schema='Attachment',
)['json']
self.records[attachment_id].append(
(time.monotonic(), meta['status'],
meta.get('ready_pipeline_version'))
)
except Exception:
pass
self.stop_event.wait(0.2)
def stop(self):
self.stop_event.set()
self.join(timeout=5)
def build_message(status, job, attempt=None):
created = next(
event for event in status['events']
if event['job_id'] == job['job_id'] and event['event'] == 'created'
)
return {
'schema_version': 1,
'job_id': job['job_id'],
'type': 'attachment.process.v1',
'attachment_id': status['attachment']['id'],
'pipeline_version': job['pipeline_version'],
'attempt': attempt if attempt is not None else job['attempt'],
'created_at': created['created_at'],
}
def publish_raw(messages):
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
for message in messages)
code, payload = cli('publish-raw', stdin=text)
check(code == 0, 'publish-raw failed: ' + str(payload))
return payload
def status_of(attachment_id):
code, payload = cli('status', attachment_id)
check(code == 0, 'status failed: ' + str(payload))
return payload
def flush_outbox(attachment_id):
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
30, 'outbox publish', 1.0)
def new_attachment_id(chat_id, before):
rows = cli('find-by-chat', chat_id)[1]
for row in reversed(rows):
if row['id'] not in before:
return row['id']
return None
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
compare_baseline=True):
if attachment_id is None:
return {'I1': True, 'I2': True, 'I3': True,
'I4': True, 'I5': True, 'I6': True}
status = status_of(attachment_id)
attachment = status['attachment']
jobs = status['jobs']
variants = status['variants']
ready_version = attachment['ready_pipeline_version']
records = tracker.records.get(attachment_id, [])
reached = any(record[1] == ('failed' if expect_failed else 'ready')
for record in records)
if not records:
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
i1 = reached or attachment['status'] in ('ready', 'failed')
ready_variants = [v for v in variants
if v['pipeline_version'] == ready_version]
succeeded_ids = {job['job_id'] for job in jobs
if job['status'] == 'succeeded'}
succeeded_events = {}
for event in status['events']:
if event['event'] == 'succeeded':
succeeded_events[event['job_id']] = (
succeeded_events.get(event['job_id'], 0) + 1
)
i2 = (len(ready_variants) == 3
and all(succeeded_events.get(job_id) == 1
for job_id in succeeded_ids))
i3 = True
seen_ready = False
last_version = 0
for _at, state, version in records:
if seen_ready and state != 'ready':
i3 = False
if state == 'ready':
seen_ready = True
if version is not None:
if version < last_version:
i3 = False
last_version = version
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
keys = {(variant['pipeline_version'], variant['variant'])
for variant in variants}
i5 = len(keys) == len(variants)
if compare_baseline and ready_version == 1 and ready_variants:
observed = {variant['variant']: variant['sha256']
for variant in ready_variants}
i6 = all(observed.get(name) == digest
for name, digest in baseline.items())
else:
i6 = True
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
'I5': i5, 'I6': i6}
# --- scenarios -----------------------------------------------------------
def scenario_duplicate(ctx):
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'duplicate.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
status = status_of(attachment['id'])
before = {variant['variant']: variant['s3']
for variant in status['variants']}
message = build_message(status, status['jobs'][0])
publish_raw([message] * 5)
wait_for(
lambda: len([event for event in status_of(attachment['id'])['events']
if event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded']) == 5,
20, 'five duplicate_skipped events', 1.0,
)
after_status = status_of(attachment['id'])
after = {variant['variant']: variant['s3']
for variant in after_status['variants']}
check(before == after, 'duplicate delivery rewrote variant objects')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
def scenario_reorder(ctx):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'reorder.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
flush_outbox(attachment['id'])
status = status_of(attachment['id'])
message = build_message(status, status['jobs'][0])
stale = dict(message, attempt=99)
publish_raw([stale, message])
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
finally:
compose('start', 'worker')
after_status = status_of(attachment['id'])
skipped = [event for event in after_status['events']
if event['event'] == 'duplicate_skipped']
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
check(len([event for event in after_status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one successful effect')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
def crash_scenario(ctx, point, filename, expected_attempt,
expect_lease_expired=False, expect_duplicate=False):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), filename)
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
starts = worker_start_count()
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
finally:
compose('start', 'worker')
status = status_of(attachment['id'])
job = status['jobs'][0]
check(job['attempt'] == expected_attempt,
'expected attempt ' + str(expected_attempt) +
', got ' + str(job['attempt']))
check(worker_start_count() >= starts + 2,
'worker process did not restart after the injected crash')
if expect_lease_expired:
check(any(event['event'] == 'lease_expired'
for event in status['events']),
'missing lease_expired event')
if expect_duplicate:
check(any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded'
for event in status['events']),
'missing already_succeeded duplicate after redelivery')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt']],
'notes': 'crashed at ' + point}
def scenario_crash_before_side_effect(ctx):
return crash_scenario(ctx, 'worker.before_side_effect',
'crash-before.png', 2, expect_lease_expired=True)
def scenario_crash_after_side_effect(ctx):
return crash_scenario(ctx, 'worker.after_side_effect',
'crash-after.png', 2, expect_lease_expired=True)
def scenario_crash_after_commit_before_ack(ctx):
return crash_scenario(ctx, 'worker.after_commit_before_ack',
'crash-ack.png', 1, expect_duplicate=True)
def scenario_crash_after_commit_before_publish(ctx):
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
set_fault('fault:api.after_commit_before_publish', 'crash')
try:
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=crash-publish.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# A killed replica may close the connection instead of answering.
pass
finally:
clear_fault('fault:api.after_commit_before_publish')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after API crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'created']) == 1,
'expected exactly one created event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'outbox relay published the committed job after API restart'}
def scenario_relay_crash_after_publish(ctx):
compose('stop', 'worker')
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
try:
set_fault('fault:relay.after_publish_before_mark', 'crash')
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=relay-crash.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# The relay may crash the handling replica before the reply flushes.
pass
clear_fault('fault:relay.after_publish_before_mark')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after relay crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
60, 'outbox published after relay crash', 1.0)
queue_depth = rabbit_queues().get('attachments.process', {}).get(
'messages', 0)
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
clear_fault('fault:relay.after_publish_before_mark')
compose('start', 'worker')
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one succeeded event')
duplicated = any(event['event'] == 'duplicate_skipped'
for event in status['events'])
note = ('relay republished the same message; duplicate absorbed'
if duplicated else
'relay published once in this run; I1/I2 still hold')
if queue_depth >= 2:
check(duplicated, 'two queued copies but no duplicate_skipped event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': note}
def scenario_stale_version(ctx):
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
'stale_version needs WORKER_CONCURRENCY>=2')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'stale.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
flush_outbox(attachment_id)
code, payload = cli('reprocess', attachment_id,
'--pipeline-version', '2', '--reason', 'suite')
check(code == 0, 'reprocess v2 failed: ' + str(payload))
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:8000')
compose('start', 'worker')
v2_etags = None
def both_terminal():
nonlocal v2_etags
status = status_of(attachment_id)
jobs = {(job['pipeline_version']): job for job in status['jobs']}
if '2' in jobs or 2 in jobs:
v2_job = jobs.get(2)
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
v2_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
v1_job = jobs.get(1)
v2_job = jobs.get(2)
return (v1_job and v2_job
and v1_job['status'] == 'superseded'
and v2_job['status'] == 'succeeded')
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
status = status_of(attachment_id)
check(status['attachment']['ready_pipeline_version'] == 2,
'ready pipeline version is not 2')
check(all(variant['pipeline_version'] == 2
for variant in status['variants']),
'variant rows from v1 survived')
after_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
served = ctx['client'].call(
'GET',
'/api/v1/attachments/' + attachment_id +
'/content?variant=thumbnail',
actor=ctx['alice'],
)
v2_thumbnail = next(variant for variant in status['variants']
if variant['variant'] == 'thumbnail')
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
'served thumbnail is not the v2 object')
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
'served thumbnail equals the v1 baseline')
v1_job = next(job for job in status['jobs']
if job['pipeline_version'] == 1)
publish_raw([build_message(status, v1_job)])
wait_for(
lambda: any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'superseded'
for event in status_of(attachment_id)['events']),
20, 'superseded duplicate skip', 1.0,
)
finally:
clear_fault('fault:worker.after_side_effect:' +
(attachment['id'] if 'attachment' in locals() else ''))
compose('start', 'worker')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'v1 finished after v2 and was recorded as superseded'}
def scenario_poison(ctx):
before = rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0)
publish_raw(['not-json', '{"schema_version":1}'])
wait_for(
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0) >= before + 2,
15, 'two poison messages in DLQ', 0.5,
)
return {'attachment_id': None, 'job_attempts': [],
'notes': 'malformed messages moved to the DLQ without touching jobs'}
def scenario_retry_and_replay(ctx):
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'retry.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
'transient')
compose('start', 'worker')
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
90, allow_failed=True)
finally:
compose('start', 'worker')
check(failed['status'] == 'failed', 'transient fault was not exhausted')
check(failed['error_code'] == 'retry_exhausted',
'expected retry_exhausted, got ' + str(failed['error_code']))
status = status_of(attachment_id)
retries = [event for event in status['events']
if event['event'] == 'retry_scheduled']
check(len(retries) == max_attempts - 1,
'expected ' + str(max_attempts - 1) + ' retries, got ' +
str(len(retries)))
for index, event in enumerate(retries):
limit = min(cap, base * 2 ** index)
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
'retry delay exceeds its jitter cap')
ctx['client'].call(
'GET', '/api/v1/attachments/' + attachment_id + '/content',
409, actor=ctx['alice'],
)
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
check(code == 3 and payload.get('reason') in ('no_failed_job',
'already_ready'),
'replay of a ready attachment was not refused')
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
code, payload = cli('replay', attachment_id, '--reason', 'suite')
check(code == 0, 'operator replay failed: ' + str(payload))
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
status = status_of(attachment_id)
replayed = [event for event in status['events']
if event['event'] == 'replayed']
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
'missing replayed event or operator audit')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'transient injected fault exhausted retries; replay recovered'}
def scenario_hung_worker(ctx):
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'hung.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:' + str((timeout_seconds + 5) * 1000))
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
compose('start', 'worker')
status = status_of(attachment_id)
check(any(event['event'] == 'retry_scheduled'
and event['detail'].get('code') == 'processing_timeout'
for event in status['events']),
'missing processing_timeout retry')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'hard timeout aborted the hung render and retried'}
def scenario_big_image(ctx):
ctx['compare_baseline'] = False
data = worst_decode_png()
containers = compose('ps', '-q', 'worker').stdout.split()
stop = threading.Event()
peak = {'mb': 0.0}
def sample():
while not stop.is_set():
if containers:
result = docker(
'stats', '--no-stream', '--format',
'{{.Name}} {{.MemUsage}}', *containers,
check_result=False,
)
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) < 2:
continue
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
if not match:
continue
value = float(match.group(1))
if match.group(2) == 'GiB':
value *= 1024
elif match.group(2) == 'KiB':
value /= 1024
peak['mb'] = max(peak['mb'], value)
stop.wait(0.3)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
try:
def do_upload(index):
client = new_client(ctx['base_url'])
return upload(client, ctx['alice'], ctx['chat'], data,
'big-' + str(index) + '.png')
with ThreadPoolExecutor(max_workers=2) as pool:
attachments = list(pool.map(do_upload, (1, 2)))
ctx['attachment_id'] = attachments[0]['id']
for attachment in attachments:
ctx['tracker'].track(attachment['id'])
results = [wait_ready(ctx['client'], ctx['alice'],
attachment['id'], 120, allow_failed=True)
for attachment in attachments]
finally:
stop.set()
sampler.join(timeout=5)
for meta in results:
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
if meta['status'] == 'failed':
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
'unexpected big-image error code')
for container in containers:
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
check_result=False)
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
return {'attachment_id': attachments[0]['id'],
'job_attempts': [],
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
SCENARIOS = {
'duplicate': scenario_duplicate,
'reorder': scenario_reorder,
'crash_before_side_effect': scenario_crash_before_side_effect,
'crash_after_side_effect': scenario_crash_after_side_effect,
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
'relay_crash_after_publish': scenario_relay_crash_after_publish,
'stale_version': scenario_stale_version,
'poison': scenario_poison,
'retry_and_replay': scenario_retry_and_replay,
'hung_worker': scenario_hung_worker,
'big_image': scenario_big_image,
}
def select(name):
if name == 'all':
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
if name == 'level4':
return LEVEL4
if name == 'track2':
return TRACK2
if name in SCENARIOS:
return [name]
raise SystemExit('unknown scenario: ' + name)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=None)
parser.add_argument('--ca-file', default=None)
parser.add_argument('--scenario', default='level4')
parser.add_argument('--repeat', type=int, default=1)
parser.add_argument('--output', default=None)
args = parser.parse_args()
if worker_env('FAULT_INJECTION_ENABLED') != '1':
print('fault injection is disabled; restart the stand with:\n ' + HINT,
file=sys.stderr)
raise SystemExit(2)
client = new_client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
clear_faults()
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
ready = wait_ready(client, alice, reference['id'], 60)
baseline = variant_sha(ready)
selected = select(args.scenario)
results = []
failed = 0
for name in selected:
for repeat in range(1, args.repeat + 1):
clear_faults()
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
args.ca_file)
tracker.start()
started = time.monotonic()
ctx = {
'client': client,
'base_url': args.base_url or 'http://localhost:8080',
'alice': alice,
'chat': chat,
'tracker': tracker,
'baseline': baseline,
'reference_id': reference['id'],
'ca_file': args.ca_file,
}
scenario = {'attachment_id': None, 'job_attempts': [],
'notes': ''}
passed = True
try:
scenario.update(SCENARIOS[name](ctx))
except Exception as error: # noqa: BLE001 - scenario boundary
passed = False
scenario['notes'] = type(error).__name__ + ': ' + str(error)
finally:
tracker.stop()
clear_faults()
compose('start', 'worker', check_result=False)
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
try:
invariants = evaluate(
tracker, attachment_id, baseline,
compare_baseline=ctx.get('compare_baseline', True),
)
except Exception: # noqa: BLE001
invariants = {key: False for key in
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
passed = passed and all(invariants.values())
if not passed:
failed += 1
result = {
'name': name,
'repeat': repeat,
'passed': passed,
'duration_s': round(time.monotonic() - started, 3),
'attachment_id': attachment_id,
'job_attempts': scenario['job_attempts'],
'events_summary': scenario.get('events_summary', {}),
'invariants': invariants,
'notes': scenario['notes'],
}
results.append(result)
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
' ' + str(result['duration_s']) + 's ' +
json.dumps(invariants) + ' ' + result['notes'],
flush=True)
if args.output:
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
raise SystemExit(1 if failed else 0)
if __name__ == '__main__':
main()
+180
View File
@@ -0,0 +1,180 @@
"""Shared stdlib-only helpers for the lab5 demo and fault suite."""
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
ENV_FILE = '.env'
COMPOSE = ['docker', 'compose', '--env-file', ENV_FILE]
def compose(*args, stdin=None, check_result=True):
result = subprocess.run(
COMPOSE + list(args),
cwd=ROOT,
input=stdin,
text=True,
capture_output=True,
)
if check_result and result.returncode != 0:
detail = (result.stderr or result.stdout).strip().splitlines()
raise Failure('compose ' + ' '.join(args) + ' failed: ' +
(detail[-1] if detail else 'unknown error'))
return result
def docker(*args, check_result=True):
result = subprocess.run(
['docker'] + list(args), cwd=ROOT, text=True, capture_output=True,
)
if check_result and result.returncode != 0:
detail = (result.stderr or result.stdout).strip().splitlines()
raise Failure('docker ' + ' '.join(args) + ' failed: ' +
(detail[-1] if detail else 'unknown error'))
return result
def cli(*args, stdin=None):
"""Run jobs.cli in the API container; returns (exit_code, parsed_json)."""
for _ in range(5):
result = compose(
'exec', '-T', 'api', 'node', 'dist/jobs/jobs.cli.js', *args,
stdin=stdin, check_result=False,
)
text = result.stdout.strip()
if text:
try:
return result.returncode, json.loads(text)
except ValueError:
pass
time.sleep(1)
raise Failure('jobs.cli did not return JSON for ' + ' '.join(args))
def set_fault(key, value):
compose('exec', '-T', 'redis', 'redis-cli', 'SET', key, value)
def clear_fault(key):
compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key)
def clear_faults():
script = (
"local keys=redis.call('keys',ARGV[1]) "
"for i=1,#keys do redis.call('del',keys[i]) end return #keys"
)
compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', script, '0', 'fault:*')
def rabbit_queues():
result = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q',
'-p', 'messenger', 'list_queues',
'name', 'messages', 'type')
queues = {}
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0] not in ('name', 'Timeout:'):
try:
queues[parts[0]] = {'messages': int(parts[1]),
'type': parts[2] if len(parts) > 2 else ''}
except ValueError:
continue
return queues
def wait_for(predicate, timeout, description='condition', interval=0.5):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = predicate()
if last:
return last
time.sleep(interval)
raise Failure('timed out waiting for ' + description)
def upload(client, actor, chat, data, filename='sample.png',
content_type='image/png'):
result = client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
202,
actor=actor,
raw=data,
headers={'Content-Type': content_type},
schema='Attachment',
)
check(result['json']['status'] == 'queued', 'upload did not answer queued')
return result['json']
def metadata(client, actor, attachment_id):
return client.call('GET', '/api/v1/attachments/' + attachment_id,
actor=actor, schema='Attachment')['json']
def wait_ready(client, actor, attachment_id, timeout=60,
allow_failed=False):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
meta = metadata(client, actor, attachment_id)
if meta['status'] == 'failed':
if allow_failed:
return meta
raise Failure('attachment failed: ' +
str(meta.get('error_code')))
if meta['status'] == 'ready':
return meta
time.sleep(0.2)
raise Failure('attachment did not become ready in ' + str(timeout) + 's')
def sha256(data):
return hashlib.sha256(data).hexdigest()
def variant_sha(meta):
return {item['name']: item['sha256'] for item in meta['variants']}
def sample_bytes():
return (ROOT / 'tests/fixtures/sample.png').read_bytes()
def worker_env(name):
result = compose('exec', '-T', 'worker', 'printenv', name,
check_result=False)
return result.stdout.strip()
def worker_start_count():
result = compose('logs', '--no-color', 'worker', check_result=False)
return result.stdout.count('worker ready:')
def worker_memory_peak(containers):
names = [line for line in containers.splitlines() if line.strip()]
if not names:
return 'n/a'
result = subprocess.run(
['docker', 'stats', '--no-stream', '--format',
'{{.Name}} {{.MemUsage}}'] + names,
text=True, capture_output=True,
)
return result.stdout.strip()
def print_step(number, text):
print('[' + str(number) + '] ' + text, flush=True)
def new_client(base_url, ca_file=None):
return Client(base_url, ca_file)
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Level-3 demo: work accepted while the worker is down survives restarts."""
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from stand import ( # noqa: E402
Failure,
check,
cli,
compose,
new_client,
print_step,
rabbit_queues,
sample_bytes,
sha256,
upload,
wait_for,
wait_ready,
)
from client import arguments, run # noqa: E402
from image_probe import jpeg_size # noqa: E402
def main():
args = arguments('lab5 worker-down durability demo')
client = new_client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
data = sample_bytes()
print_step(1, 'compose ps')
print(compose('ps', '--format',
'table {{.Service}}\t{{.Status}}').stdout.strip())
print_step(2, 'stop worker')
compose('stop', 'worker')
print_step(3, 'upload while worker is down (expect 202 queued)')
attachment = upload(client, alice, chat, data, 'durability.png')
attachment_id = attachment['id']
print('attachment=' + attachment_id + ' status=' + attachment['status'])
print_step(4, 'content and thumbnail must answer 409 before ready')
for query in ('', '?variant=thumbnail'):
result = client.call(
'GET',
'/api/v1/attachments/' + attachment_id + '/content' + query,
409,
actor=alice,
)
check(result['status'] == 409, 'content was served before ready')
print_step(5, 'link the queued attachment to messages')
with_attachment = client.message(
alice, chat, '', attachments=[attachment_id],
)['json']
plain = client.message(alice, chat, 'plain while worker is down')['json']
check(with_attachment['attachment_ids'] == [attachment_id],
'message lost the queued attachment')
check(plain['attachment_ids'] == [], 'plain message gained an attachment')
print('message=' + with_attachment['id'])
print_step(6, 'wait for outbox publish and a message in the queue')
def published():
_code, status = cli('status', attachment_id)
queues = rabbit_queues()
return (
status['outbox']['pending'] == 0
and queues.get('attachments.process', {}).get('messages', 0) >= 1
)
wait_for(published, 30, 'outbox publish and queue depth >= 1')
queues = rabbit_queues()
print('queue=' + str(queues.get('attachments.process')))
print_step(7, 'restart broker and API with volumes')
compose('restart', 'broker', 'api')
compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api')
print_step(8, 'the quorum message survived the broker restart')
def survived():
depth = rabbit_queues().get('attachments.process', {}).get('messages', 0)
return depth >= 1
wait_for(survived, 60, 'queue depth >= 1 after broker restart')
print('queue=' + str(rabbit_queues().get('attachments.process')))
print_step(9, 'start worker')
compose('start', 'worker')
print_step(10, 'wait ready and verify the thumbnail')
ready = wait_ready(client, alice, attachment_id, 60)
thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
content = client.call(
'GET',
'/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail',
actor=alice,
)
check(jpeg_size(content['bytes']) == (256, 160),
'thumbnail is not 256x160 JPEG')
check(len(content['bytes']) == thumbnail['size_bytes'],
'thumbnail size differs from metadata')
check(sha256(content['bytes']) == thumbnail['sha256'],
'thumbnail checksum differs from metadata')
history = client.call(
'GET',
'/api/v1/chats/' + chat['id'] + '/messages?limit=100',
actor=alice,
schema='MessagePage',
)['json']
check(
any(attachment_id in item['attachment_ids']
for item in history['items']),
'message history lost the attachment',
)
print('ready attachment=' + attachment_id +
' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...')
print_step(11, 'compose ps')
print(compose('ps', '--format',
'table {{.Service}}\t{{.Status}}').stdout.strip())
print('PASS: worker-down durability demo')
if __name__ == '__main__':
run(main)