Files
mtusi2026lab6/student-tests/pipeline_ops.py
T

487 lines
26 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Сценарии асинхронной обработки lab5 на Docker-стенде.
demo уровень 3: worker остановлен → upload 202 → сообщение с вложением →
content 409, чат работает → рестарт API и broker (volume) → worker → ready
outbox-gap relay остановлен (разрыв COMMIT↔publish) → upload → relay → ready;
relay падает после confirm до отметки outbox → повторная публикация → один результат
crash-effect worker падает после записи объектов до COMMIT → lease истекает → повтор → тот же результат
crash-commit worker падает после COMMIT до ack → повторная доставка → no-op
broker-restart принятая задача в quorum-очереди переживает рестарт RabbitMQ с volume
transient S3 недоступен во время обработки → retry с backoff и jitter → ready
failed-replay постоянная ошибка → failed (error_code), реестр, явный replay, повтор успешной запрещён;
невалидное сообщение → DLQ
limits изображение 36 Мпикс и параллельная обработка: пик памяти worker, время
versions трек 5: reprocess v2, устаревший дубль v1 и повтор v2 после публикации не меняют результат
Каждый сценарий печатает JSON-итог и PASS/FAIL; ID вложений и задания — без токенов.
"""
import hashlib
import json
import struct
import time
import zlib
from labutil import (BUCKET, bucket_count, compose, fixture, mc, psql, sh, wait_status)
from client import Client, Failure, arguments, check, run
from image_probe import jpeg_size
SAMPLE = None
def sample():
global SAMPLE
if SAMPLE is None:
from pathlib import Path
SAMPLE = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
return SAMPLE
def upload(c, actor, chat, data, name='p.png', ctype='image/png'):
r = c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=actor,
raw=data, headers={'Content-Type': ctype}, schema='Attachment')
check(r['json']['status'] == 'queued', 'upload snapshot must be queued')
return r['json']
def wait_ready_api(c, timeout=90):
end = time.time() + timeout
while time.time() < end:
try:
c.call('GET', '/health/ready')
return
except Failure:
time.sleep(1)
raise Failure('API not ready')
def job_of(att_id, version=1):
row = psql("SELECT job_id || '|' || state || '|' || attempt FROM jobs WHERE attachment_id = '%s' AND pipeline_version = %d"
% (att_id, version))
jid, state, attempt = row.split('|')
return jid, state, int(attempt)
def events(job_id):
return psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE job_id = '%s'" % job_id)
def invariants(att_id, chat_id):
"""I1 ≤1 варианта на (версия, имя); I2 ready ⇒ 3 варианта опубликованной версии с объектами;
I3 объектов v/ ровно 3 на каждую версию с вариантами (повторы не размножают)."""
dup = psql("SELECT count(*) FROM (SELECT 1 FROM attachment_variants WHERE attachment_id='%s' "
"GROUP BY pipeline_version, name HAVING count(*) > 1) d" % att_id)
check(dup == '0', 'I1 duplicated variants')
st, ver = psql("SELECT status || '|' || pipeline_version FROM attachments WHERE id='%s'" % att_id).split('|')
out = {'status': st, 'published_version': int(ver)}
if st == 'ready':
n = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s' AND pipeline_version=%s" % (att_id, ver))
check(n == '3', 'I2 ready with %s variants' % n)
versions = psql("SELECT string_agg(DISTINCT pipeline_version::text, ',') FROM attachment_variants WHERE attachment_id='%s'" % att_id)
for v in versions.split(','):
objs = bucket_count('v/%s/%s/p%s/' % (chat_id, att_id, v))
check(objs == 3, 'I3 version %s has %d objects' % (v, objs))
out['objects_per_version'] = {v: 3 for v in versions.split(',')}
return out
def check_variants(c, actor, meta, original):
vs = {v['name']: v for v in meta['variants']}
check(set(vs) == {'thumbnail', 'crop', 'optimized'}, 'variants: %s' % sorted(vs))
th = c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % meta['id'], actor=actor)
check(th['headers'].get_content_type() == 'image/jpeg', 'thumbnail type')
check(hashlib.sha256(th['bytes']).hexdigest() == vs['thumbnail']['sha256'], 'thumbnail sha')
tw, thh = jpeg_size(th['bytes'])
cr = c.call('GET', '/api/v1/attachments/%s/content?variant=crop' % meta['id'], actor=actor)
check(jpeg_size(cr['bytes']) == (128, 128), 'crop size')
op = c.call('GET', '/api/v1/attachments/%s/content?variant=optimized' % meta['id'], actor=actor)
check(op['bytes'][:4] == b'RIFF' and op['bytes'][8:12] == b'WEBP', 'optimized is not WebP')
check(hashlib.sha256(op['bytes']).hexdigest() == vs['optimized']['sha256'], 'optimized sha')
orig = c.call('GET', '/api/v1/attachments/%s/content' % meta['id'], actor=actor)
check(orig['bytes'] == original, 'original changed')
return {'thumbnail': [tw, thh], 'crop': [128, 128],
'optimized': [vs['optimized']['width'], vs['optimized']['height']],
'optimized_bytes': vs['optimized']['size_bytes'], 'original_bytes': len(original)}
def rabbit_healthy(timeout=90):
end = time.time() + timeout
while time.time() < end:
st = sh('docker', 'inspect', '-f', '{{.State.Health.Status}}', compose('ps', '-q', 'rabbitmq').stdout.strip(),
check=False).stdout.strip()
if st == 'healthy':
return
time.sleep(1)
raise Failure('rabbitmq not healthy')
def queues():
"""Сообщений в очередях RabbitMQ (без строки заголовка)."""
rabbit_healthy()
out = compose('exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', '--no-table-headers',
'name', 'messages').stdout
res = {}
for line in out.splitlines():
parts = line.split()
if len(parts) == 2 and parts[1].isdigit():
res[parts[0]] = int(parts[1])
return res
def drain(timeout=180):
"""Изоляция сценариев: все сервисы pipeline запущены, незавершённых заданий нет."""
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 'rabbitmq', 's3', timeout=240)
end = time.time() + timeout
while time.time() < end:
busy = psql("SELECT count(*) FROM jobs WHERE state IN ('queued', 'processing')")
if busy == '0':
return
time.sleep(1)
raise Failure('pipeline did not drain: %s jobs pending' % busy)
def recreate(service, **env):
"""Пересоздать сервис с переменными окружения (точки сбоя, таймауты)."""
compose('up', '-d', '--no-deps', '--wait', service, env=env, timeout=180)
# ---------------- уровень 3 ----------------
def scenario_demo(c):
alice, bob, _, chat = c.bootstrap()
timeline = []
compose('stop', 'worker')
timeline.append('worker stopped')
att = upload(c, alice, chat, sample(), name='demo.png')
timeline.append('upload 202 queued id=%s' % att['id'])
msg = c.message(alice, chat, 'с вложением в очереди', attachments=[att['id']])['json']
check(msg['attachment_ids'] == [att['id']], 'message attachment')
timeline.append('message 201 with queued attachment')
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 409, actor=bob)
c.message(bob, chat, 'чат работает без worker')
timeline.append('content 409, other messages OK')
time.sleep(2)
published = psql("SELECT count(*) FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att['id'])
compose('restart', 'api')
compose('restart', 'rabbitmq')
sh('sh', 'scripts/restore.sh')
wait_ready_api(c)
timeline.append('api and rabbitmq restarted (volumes kept), outbox published=%s' % published)
q = {}
for _ in range(30):
q = queues()
if q.get('attachment.process', 0) >= 1:
break
time.sleep(1)
timeline.append('rabbitmq queues after restart: %s' % q)
check(q.get('attachment.process', 0) >= 1, 'accepted job not in broker after restart: %s' % q)
t0 = time.time()
compose('start', 'worker')
meta = wait_status(c, bob, att['id'], ('ready',), timeout=90)
timeline.append('worker started → ready in %.1f s' % (time.time() - t0))
res = check_variants(c, bob, meta, sample())
check(res['thumbnail'] == [256, 160], 'fixture thumbnail must be 256x160')
jid, state, attempt = job_of(att['id'])
return {'attachment_id': att['id'], 'job_id': jid, 'job_state': state, 'attempt': attempt,
'timeline': timeline, 'variants': res, 'invariants': invariants(att['id'], chat['id']),
'events': events(jid)}
# ---------------- уровень 4 ----------------
def scenario_outbox_gap(c):
alice, bob, _, chat = c.bootstrap()
compose('stop', 'relay')
att = upload(c, alice, chat, sample(), name='gap.png')
time.sleep(3)
jid, state, _ = job_of(att['id'])
unpublished = psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NULL" % jid)
check(state == 'queued' and unpublished == '1', 'job must wait in outbox: %s %s' % (state, unpublished))
compose('restart', 'api') # «падение» после COMMIT, publish ещё не было
wait_ready_api(c)
compose('start', 'relay')
meta = wait_status(c, bob, att['id'], ('ready',))
part1 = {'attachment_id': att['id'], 'unpublished_while_relay_down': 1, 'then': meta['status']}
# Relay падает ПОСЛЕ publisher confirm, ДО отметки published_at → повторная публикация.
recreate('relay', RELAY_FAULT_POINTS='relay.after_publish')
att2 = upload(c, alice, chat, sample(), name='dup.png')
meta2 = wait_status(c, bob, att2['id'], ('ready',), timeout=120)
jid2, _, attempt2 = job_of(att2['id'])
for _ in range(60):
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid2) == '1':
break
time.sleep(0.5)
attempts = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid2)
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', compose('ps', '-q', 'relay').stdout.strip()).stdout.strip()
recreate('relay')
worker_log = compose('logs', '--no-color', 'worker').stdout
dup_seen = worker_log.count(jid2) >= 2
check(int(restarts) >= 1, 'fault relay.after_publish did not crash the relay')
check(attempt2 == 1, 'duplicate publish must not create a second attempt')
part2 = {'attachment_id': att2['id'], 'relay_restarts': int(restarts), 'outbox_publish_attempts': int(attempts or 0),
'worker_saw_message_twice': dup_seen, 'job_attempt': attempt2, 'invariants': invariants(att2['id'], chat['id'])}
return {'relay_down_then_up': part1, 'relay_crash_after_confirm': part2, 'events': events(jid2)}
def scenario_crash_effect(c):
alice, bob, _, chat = c.bootstrap()
recreate('worker', WORKER_FAULT_POINTS='worker.after_side_effect', JOB_TIMEOUT_SECONDS='15')
t0 = time.time()
att = upload(c, alice, chat, sample(), name='crash.png')
meta = wait_status(c, bob, att['id'], ('ready',), timeout=120)
took = time.time() - t0
jid, state, attempt = job_of(att['id'])
cid = compose('ps', '-q', 'worker').stdout.split()[0]
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid).stdout.strip()
recreate('worker')
check(attempt == 2, 'expected recovery on attempt 2, got %d; events=%s' % (attempt, events(jid)))
ev = events(jid)
check('lease_expired_requeued' in ev, 'lease expiry recovery missing: ' + ev)
return {'attachment_id': att['id'], 'job_id': jid, 'worker_restarts': int(restarts), 'attempt': attempt,
'seconds_to_ready': round(took, 1), 'events': ev, 'invariants': invariants(att['id'], chat['id']),
'variants': check_variants(c, bob, meta, sample())}
def scenario_crash_commit(c):
alice, bob, _, chat = c.bootstrap()
recreate('worker', WORKER_FAULT_POINTS='worker.after_commit')
att = upload(c, alice, chat, sample(), name='commit.png')
wait_status(c, bob, att['id'], ('ready',))
time.sleep(8) # контейнер перезапускается, broker доставляет неподтверждённое сообщение повторно
jid, state, attempt = job_of(att['id'])
cid = compose('ps', '-q', 'worker').stdout.split()[0]
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid).stdout.strip()
log = compose('logs', '--no-color', 'worker').stdout
noop = any(jid in l and 'duplicate delivery of finished job' in l for l in log.splitlines())
recreate('worker')
check(int(restarts) >= 1, 'fault worker.after_commit did not crash the worker')
check(attempt == 1 and state == 'done', 'redelivery after commit must be a no-op: %s %d' % (state, attempt))
return {'attachment_id': att['id'], 'job_id': jid, 'worker_restarts': int(restarts), 'attempt': attempt,
'redelivery_acked_as_noop': noop, 'invariants': invariants(att['id'], chat['id'])}
def scenario_broker_restart(c):
alice, bob, _, chat = c.bootstrap()
compose('stop', 'worker')
att = upload(c, alice, chat, sample(), name='broker.png')
jid, _, _ = job_of(att['id'])
for _ in range(30):
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
break
time.sleep(0.5)
before = queues()
compose('restart', 'rabbitmq')
# Quorum-очередь после рестарта выбирает лидера и восстанавливает журнал: ждём.
after = {}
for _ in range(30):
after = queues()
if after.get('attachment.process', 0) >= 1:
break
time.sleep(1)
sh('docker', 'start', compose('ps', '-aq', 'worker').stdout.split()[0]) # без запуска зависимостей
wait_status(c, bob, att['id'], ('ready',))
published = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid)
check(after.get('attachment.process', 0) >= 1, 'message lost on broker restart: %s' % after)
check(published == '1', 'message was republished from outbox (%s): not a broker durability proof' % published)
return {'attachment_id': att['id'], 'queue_before_restart': before, 'queue_after_restart': after,
'outbox_publish_attempts': int(published),
'invariants': invariants(att['id'], chat['id'])}
def scenario_transient(c):
alice, bob, _, chat = c.bootstrap()
worker_cid = compose('ps', '-q', 'worker').stdout.split()[0]
s3_cid = compose('ps', '-q', 's3').stdout.split()[0]
compose('stop', 'worker')
att = upload(c, alice, chat, sample(), name='retry.png')
jid, _, _ = job_of(att['id'])
for _ in range(30):
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
break
time.sleep(0.5)
# Прямые docker stop/start: `docker compose start worker` поднял бы и S3
# (worker → s3-init → s3), и временного отказа бы не было.
sh('docker', 'stop', s3_cid)
sh('docker', 'start', worker_cid)
seen = []
t0 = time.time()
while time.time() - t0 < 25:
ev = events(jid) or ''
seen = ev.split(',')
if seen.count('retry_scheduled') >= 2:
break
time.sleep(1)
sh('docker', 'start', s3_cid)
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
_, state, attempt = job_of(att['id'])
details = psql("SELECT string_agg(detail, ' | ' ORDER BY id) FROM job_events WHERE job_id='%s' AND event='retry_scheduled'" % jid)
check(attempt >= 2 and state == 'done', 'expected retries then success: %s/%d events=%s' % (state, attempt, events(jid)))
return {'attachment_id': att['id'], 'attempts': attempt, 'final_status': meta['status'],
'retry_details': details, 'events': events(jid), 'invariants': invariants(att['id'], chat['id'])}
def scenario_failed_replay(c):
alice, bob, _, chat = c.bootstrap()
compose('stop', 'worker')
att = upload(c, alice, chat, sample(), name='bad.png')
key = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
mc('printf garbage > /tmp/g && mc cp -q /tmp/g s3/%s/%s >/dev/null' % (BUCKET, key)) # порча staging
compose('start', 'worker')
meta = wait_status(c, bob, att['id'], ('failed',))
check(meta['error_code'] == 'checksum_mismatch', 'error_code %s' % meta['error_code'])
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 409, actor=bob)
c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 409, actor=alice,
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': 'f-' + att['id']})
registry = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'failed').stdout
check(att['id'] in registry, 'failed registry must list the job')
# Восстанавливаем исходник и делаем явный replay.
import subprocess
p = subprocess.run(['docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 's3-init', '-c',
'mc alias set s3 http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null; '
'cat > /tmp/o && mc cp -q /tmp/o s3/%s/%s >/dev/null' % (BUCKET, key)], input=sample(), capture_output=True)
check(p.returncode == 0, 'restore staging failed')
rp = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'replay', att['id'],
'--actor', 'lab5-test', check=False)
check(rp.returncode == 0, 'replay failed: ' + rp.stdout + rp.stderr[-200:])
ready = wait_status(c, bob, att['id'], ('ready',))
again = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'replay', att['id'],
'--actor', 'lab5-test', check=False)
check(again.returncode != 0, 'replay of a finished job must be refused')
jid, _, attempt = job_of(att['id'])
# Невалидное сообщение → DLQ (не крутится вечно, не теряется молча).
psql("INSERT INTO outbox (job_id, payload) VALUES ('%s', '{\"schema_version\": 99}'::jsonb)" % jid)
dead = 0
for _ in range(30):
dead = queues().get('attachment.dead', 0)
if dead >= 1:
break
time.sleep(1)
check(dead >= 1, 'invalid message did not reach DLQ')
return {'attachment_id': att['id'], 'failed_error_code': meta['error_code'], 'content_while_failed': 409,
'attach_failed_to_message': 409, 'registry_listed': True, 'replay': json.loads(rp.stdout.strip().splitlines()[-1]),
'replay_after_success': json.loads(again.stdout.strip().splitlines()[-1]),
'final_status': ready['status'], 'events': events(jid), 'dlq_messages': dead,
'invariants': invariants(att['id'], chat['id'])}
def solid_png(w, h, rgb=(40, 120, 200)):
row = b'\x00' + bytes(rgb) * w
raw = row * h
ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)
ch = lambda t, d: struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff)
return b'\x89PNG\r\n\x1a\n' + ch(b'IHDR', ihdr) + ch(b'IDAT', zlib.compress(raw, 9)) + ch(b'IEND', b'')
def scenario_limits(c):
alice, bob, _, chat = c.bootstrap()
big = solid_png(6000, 6000) # 36 Мпикс, сжатый файл маленький
check(len(big) < 10 * 1024 * 1024, 'test image too big')
import threading
compose('restart', 'worker') # новый cgroup → счётчики с нуля
time.sleep(3)
cid = compose('ps', '-q', 'worker').stdout.split()[0]
def cg(name):
return sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/' + name, check=False).stdout
anon_max = [0]
stop = threading.Event()
def sampler():
# memory.peak включает page cache (временные файлы cwebp и т. п.), поэтому
# отдельно снимаем анонимную память процесса — её и ограничивает пиксельный бюджет.
while not stop.is_set():
for line in cg('memory.stat').splitlines():
if line.startswith('anon '):
anon_max[0] = max(anon_max[0], int(line.split()[1]))
time.sleep(0.3)
th = threading.Thread(target=sampler, daemon=True)
th.start()
t0 = time.time()
ids = [upload(c, alice, chat, big, name='big%d.png' % i)['id'] for i in range(3)]
done = [wait_status(c, bob, i, ('ready', 'failed'), timeout=240) for i in ids]
took = time.time() - t0
stop.set()
th.join()
peak = cg('memory.peak').strip()
limit = cg('memory.max').strip()
oom = {l.split()[0]: int(l.split()[1]) for l in cg('memory.events').splitlines() if len(l.split()) == 2}
check(all(d['status'] == 'ready' for d in done), 'big images failed: %s' % [d['error_code'] for d in done])
check(oom.get('oom_kill', 0) == 0, 'worker was OOM-killed')
th = {v['name']: (v['width'], v['height']) for v in done[0]['variants']}
check(th['thumbnail'] == (256, 256) and th['optimized'] == (1280, 1280), 'variant sizes %s' % th)
return {'image': '6000x6000 PNG (%d bytes)' % len(big), 'parallel_uploads': 3, 'seconds_all_ready': round(took, 1),
'worker_anon_memory_max_mib': round(anon_max[0] / 1048576, 1),
'worker_memory_peak_incl_page_cache_mib': round(int(peak) / 1048576, 1) if peak.isdigit() else peak,
'oom_kill_events': oom.get('oom_kill', 0),
'go_memlimit': sh('docker', 'exec', cid, 'printenv', 'GOMEMLIMIT', check=False).stdout.strip(),
'oom_events': oom.get('oom', 0),
'worker_memory_limit_mib': round(int(limit) / 1048576, 1) if limit.isdigit() else limit,
'pixel_budget': 60_000_000, 'decoded_rgba_per_image_mib': round(6000 * 6000 * 4 / 1048576, 1),
'variant_sizes': th}
# ---------------- трек 5: версии и устаревшие задания ----------------
def scenario_versions(c):
alice, bob, _, chat = c.bootstrap()
att = upload(c, alice, chat, sample(), name='ver.png')
v1 = wait_status(c, bob, att['id'], ('ready',))
th1 = {v['name']: v['sha256'] for v in v1['variants']}['thumbnail']
jid1, _, _ = job_of(att['id'], 1)
rp = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'reprocess', att['id'], '--version', '2')
jid2 = json.loads(rp.stdout.strip().splitlines()[-1])['job_id']
for _ in range(120):
if psql("SELECT pipeline_version FROM attachments WHERE id='%s'" % att['id']) == '2':
break
time.sleep(0.5)
v2 = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
th2 = {v['name']: v['sha256'] for v in v2['variants']}['thumbnail']
check(v2['status'] == 'ready' and th2 != th1, 'v2 must be published with a different thumbnail')
# Устаревший дубль v1 и повтор v2 после публикации v2.
psql("INSERT INTO outbox (job_id, payload) SELECT job_id, payload FROM outbox WHERE job_id='%s' ORDER BY id LIMIT 1" % jid1)
psql("INSERT INTO outbox (job_id, payload) SELECT job_id, payload FROM outbox WHERE job_id='%s' ORDER BY id LIMIT 1" % jid2)
time.sleep(6)
after = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
th_after = {v['name']: v['sha256'] for v in after['variants']}['thumbnail']
got = c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], actor=bob)['bytes']
check(th_after == th2 and hashlib.sha256(got).hexdigest() == th2, 'stale v1 overwrote v2')
check(psql("SELECT pipeline_version FROM attachments WHERE id='%s'" % att['id']) == '2', 'version went back')
_, s1, a1 = job_of(att['id'], 1)
_, s2, a2 = job_of(att['id'], 2)
check(a1 == 1 and a2 == 1, 'duplicates must not start new attempts')
return {'attachment_id': att['id'], 'v1_thumbnail_sha': th1, 'v2_thumbnail_sha': th2,
'after_stale_v1_and_dup_v2': th_after, 'published_version': 2,
'jobs': {'v1': {'state': s1, 'attempt': a1, 'events': events(jid1)},
'v2': {'state': s2, 'attempt': a2, 'events': events(jid2)}},
'invariants': invariants(att['id'], chat['id'])}
SCENARIOS = {'demo': scenario_demo, 'outbox-gap': scenario_outbox_gap, 'crash-effect': scenario_crash_effect,
'crash-commit': scenario_crash_commit, 'broker-restart': scenario_broker_restart,
'transient': scenario_transient, 'failed-replay': scenario_failed_replay,
'limits': scenario_limits, 'versions': scenario_versions}
def main():
def configure(p):
p.add_argument('scenario', choices=sorted(SCENARIOS))
p.add_argument('--output', default=None)
args = arguments('Lab5 pipeline scenarios on the Docker stand', configure)
c = Client(args.base_url, args.ca_file, timeout=60)
drain()
try:
result = SCENARIOS[args.scenario](c)
finally:
# Сценарий мог остановить сервисы или включить точки сбоя: возвращаем штатную конфигурацию.
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 'rabbitmq', 's3', 'api', timeout=300, check=False)
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
if args.output:
open(args.output, 'w', encoding='utf-8').write(text + '\n')
print(text)
print('PASS: ' + args.scenario)
if __name__ == '__main__':
run(main)