487 lines
26 KiB
Python
487 lines
26 KiB
Python
#!/usr/bin/env python3
|
||
"""Сценарии асинхронной обработки lab5 на Docker-стенде.
|
||
|
||
demo уровень 3: worker остановлен → upload 202 → сообщение с вложением →
|
||
content 409, чат работает → рестарт API и broker (volume) → worker → ready
|
||
outbox-gap relay остановлен (разрыв COMMIT↔publish) → upload → relay → ready;
|
||
relay падает после confirm до отметки outbox → повторная публикация → один результат
|
||
crash-effect worker падает после записи объектов до COMMIT → lease истекает → повтор → тот же результат
|
||
crash-commit worker падает после COMMIT до ack → повторная доставка → no-op
|
||
broker-restart принятая задача в quorum-очереди переживает рестарт RabbitMQ с volume
|
||
transient S3 недоступен во время обработки → retry с backoff и jitter → ready
|
||
failed-replay постоянная ошибка → failed (error_code), реестр, явный replay, повтор успешной запрещён;
|
||
невалидное сообщение → DLQ
|
||
limits изображение 36 Мпикс и параллельная обработка: пик памяти worker, время
|
||
versions трек 5: reprocess v2, устаревший дубль v1 и повтор v2 после публикации не меняют результат
|
||
|
||
Каждый сценарий печатает JSON-итог и PASS/FAIL; ID вложений и задания — без токенов.
|
||
"""
|
||
import hashlib
|
||
import json
|
||
import struct
|
||
import time
|
||
import zlib
|
||
from labutil import (BUCKET, bucket_count, compose, fixture, mc, psql, sh, wait_status)
|
||
from client import Client, Failure, arguments, check, run
|
||
from image_probe import jpeg_size
|
||
|
||
SAMPLE = None
|
||
|
||
|
||
def sample():
|
||
global SAMPLE
|
||
if SAMPLE is None:
|
||
from pathlib import Path
|
||
SAMPLE = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes()
|
||
return SAMPLE
|
||
|
||
|
||
def upload(c, actor, chat, data, name='p.png', ctype='image/png'):
|
||
r = c.call('POST', '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], name), 202, actor=actor,
|
||
raw=data, headers={'Content-Type': ctype}, schema='Attachment')
|
||
check(r['json']['status'] == 'queued', 'upload snapshot must be queued')
|
||
return r['json']
|
||
|
||
|
||
def wait_ready_api(c, timeout=90):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
try:
|
||
c.call('GET', '/health/ready')
|
||
return
|
||
except Failure:
|
||
time.sleep(1)
|
||
raise Failure('API not ready')
|
||
|
||
|
||
def job_of(att_id, version=1):
|
||
row = psql("SELECT job_id || '|' || state || '|' || attempt FROM jobs WHERE attachment_id = '%s' AND pipeline_version = %d"
|
||
% (att_id, version))
|
||
jid, state, attempt = row.split('|')
|
||
return jid, state, int(attempt)
|
||
|
||
|
||
def events(job_id):
|
||
return psql("SELECT string_agg(event, ',' ORDER BY id) FROM job_events WHERE job_id = '%s'" % job_id)
|
||
|
||
|
||
def invariants(att_id, chat_id):
|
||
"""I1 ≤1 варианта на (версия, имя); I2 ready ⇒ 3 варианта опубликованной версии с объектами;
|
||
I3 объектов v/ ровно 3 на каждую версию с вариантами (повторы не размножают)."""
|
||
dup = psql("SELECT count(*) FROM (SELECT 1 FROM attachment_variants WHERE attachment_id='%s' "
|
||
"GROUP BY pipeline_version, name HAVING count(*) > 1) d" % att_id)
|
||
check(dup == '0', 'I1 duplicated variants')
|
||
st, ver = psql("SELECT status || '|' || pipeline_version FROM attachments WHERE id='%s'" % att_id).split('|')
|
||
out = {'status': st, 'published_version': int(ver)}
|
||
if st == 'ready':
|
||
n = psql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s' AND pipeline_version=%s" % (att_id, ver))
|
||
check(n == '3', 'I2 ready with %s variants' % n)
|
||
versions = psql("SELECT string_agg(DISTINCT pipeline_version::text, ',') FROM attachment_variants WHERE attachment_id='%s'" % att_id)
|
||
for v in versions.split(','):
|
||
objs = bucket_count('v/%s/%s/p%s/' % (chat_id, att_id, v))
|
||
check(objs == 3, 'I3 version %s has %d objects' % (v, objs))
|
||
out['objects_per_version'] = {v: 3 for v in versions.split(',')}
|
||
return out
|
||
|
||
|
||
def check_variants(c, actor, meta, original):
|
||
vs = {v['name']: v for v in meta['variants']}
|
||
check(set(vs) == {'thumbnail', 'crop', 'optimized'}, 'variants: %s' % sorted(vs))
|
||
th = c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % meta['id'], actor=actor)
|
||
check(th['headers'].get_content_type() == 'image/jpeg', 'thumbnail type')
|
||
check(hashlib.sha256(th['bytes']).hexdigest() == vs['thumbnail']['sha256'], 'thumbnail sha')
|
||
tw, thh = jpeg_size(th['bytes'])
|
||
cr = c.call('GET', '/api/v1/attachments/%s/content?variant=crop' % meta['id'], actor=actor)
|
||
check(jpeg_size(cr['bytes']) == (128, 128), 'crop size')
|
||
op = c.call('GET', '/api/v1/attachments/%s/content?variant=optimized' % meta['id'], actor=actor)
|
||
check(op['bytes'][:4] == b'RIFF' and op['bytes'][8:12] == b'WEBP', 'optimized is not WebP')
|
||
check(hashlib.sha256(op['bytes']).hexdigest() == vs['optimized']['sha256'], 'optimized sha')
|
||
orig = c.call('GET', '/api/v1/attachments/%s/content' % meta['id'], actor=actor)
|
||
check(orig['bytes'] == original, 'original changed')
|
||
return {'thumbnail': [tw, thh], 'crop': [128, 128],
|
||
'optimized': [vs['optimized']['width'], vs['optimized']['height']],
|
||
'optimized_bytes': vs['optimized']['size_bytes'], 'original_bytes': len(original)}
|
||
|
||
|
||
def rabbit_healthy(timeout=90):
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
st = sh('docker', 'inspect', '-f', '{{.State.Health.Status}}', compose('ps', '-q', 'rabbitmq').stdout.strip(),
|
||
check=False).stdout.strip()
|
||
if st == 'healthy':
|
||
return
|
||
time.sleep(1)
|
||
raise Failure('rabbitmq not healthy')
|
||
|
||
|
||
def queues():
|
||
"""Сообщений в очередях RabbitMQ (без строки заголовка)."""
|
||
rabbit_healthy()
|
||
out = compose('exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', '--no-table-headers',
|
||
'name', 'messages').stdout
|
||
res = {}
|
||
for line in out.splitlines():
|
||
parts = line.split()
|
||
if len(parts) == 2 and parts[1].isdigit():
|
||
res[parts[0]] = int(parts[1])
|
||
return res
|
||
|
||
|
||
def drain(timeout=180):
|
||
"""Изоляция сценариев: все сервисы pipeline запущены, незавершённых заданий нет."""
|
||
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 'rabbitmq', 's3', timeout=240)
|
||
end = time.time() + timeout
|
||
while time.time() < end:
|
||
busy = psql("SELECT count(*) FROM jobs WHERE state IN ('queued', 'processing')")
|
||
if busy == '0':
|
||
return
|
||
time.sleep(1)
|
||
raise Failure('pipeline did not drain: %s jobs pending' % busy)
|
||
|
||
|
||
def recreate(service, **env):
|
||
"""Пересоздать сервис с переменными окружения (точки сбоя, таймауты)."""
|
||
compose('up', '-d', '--no-deps', '--wait', service, env=env, timeout=180)
|
||
|
||
|
||
# ---------------- уровень 3 ----------------
|
||
|
||
def scenario_demo(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
timeline = []
|
||
compose('stop', 'worker')
|
||
timeline.append('worker stopped')
|
||
att = upload(c, alice, chat, sample(), name='demo.png')
|
||
timeline.append('upload 202 queued id=%s' % att['id'])
|
||
msg = c.message(alice, chat, 'с вложением в очереди', attachments=[att['id']])['json']
|
||
check(msg['attachment_ids'] == [att['id']], 'message attachment')
|
||
timeline.append('message 201 with queued attachment')
|
||
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 409, actor=bob)
|
||
c.message(bob, chat, 'чат работает без worker')
|
||
timeline.append('content 409, other messages OK')
|
||
time.sleep(2)
|
||
published = psql("SELECT count(*) FROM outbox o JOIN jobs j USING (job_id) WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att['id'])
|
||
compose('restart', 'api')
|
||
compose('restart', 'rabbitmq')
|
||
sh('sh', 'scripts/restore.sh')
|
||
wait_ready_api(c)
|
||
timeline.append('api and rabbitmq restarted (volumes kept), outbox published=%s' % published)
|
||
q = {}
|
||
for _ in range(30):
|
||
q = queues()
|
||
if q.get('attachment.process', 0) >= 1:
|
||
break
|
||
time.sleep(1)
|
||
timeline.append('rabbitmq queues after restart: %s' % q)
|
||
check(q.get('attachment.process', 0) >= 1, 'accepted job not in broker after restart: %s' % q)
|
||
t0 = time.time()
|
||
compose('start', 'worker')
|
||
meta = wait_status(c, bob, att['id'], ('ready',), timeout=90)
|
||
timeline.append('worker started → ready in %.1f s' % (time.time() - t0))
|
||
res = check_variants(c, bob, meta, sample())
|
||
check(res['thumbnail'] == [256, 160], 'fixture thumbnail must be 256x160')
|
||
jid, state, attempt = job_of(att['id'])
|
||
return {'attachment_id': att['id'], 'job_id': jid, 'job_state': state, 'attempt': attempt,
|
||
'timeline': timeline, 'variants': res, 'invariants': invariants(att['id'], chat['id']),
|
||
'events': events(jid)}
|
||
|
||
|
||
# ---------------- уровень 4 ----------------
|
||
|
||
def scenario_outbox_gap(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
compose('stop', 'relay')
|
||
att = upload(c, alice, chat, sample(), name='gap.png')
|
||
time.sleep(3)
|
||
jid, state, _ = job_of(att['id'])
|
||
unpublished = psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NULL" % jid)
|
||
check(state == 'queued' and unpublished == '1', 'job must wait in outbox: %s %s' % (state, unpublished))
|
||
compose('restart', 'api') # «падение» после COMMIT, publish ещё не было
|
||
wait_ready_api(c)
|
||
compose('start', 'relay')
|
||
meta = wait_status(c, bob, att['id'], ('ready',))
|
||
part1 = {'attachment_id': att['id'], 'unpublished_while_relay_down': 1, 'then': meta['status']}
|
||
|
||
# Relay падает ПОСЛЕ publisher confirm, ДО отметки published_at → повторная публикация.
|
||
recreate('relay', RELAY_FAULT_POINTS='relay.after_publish')
|
||
att2 = upload(c, alice, chat, sample(), name='dup.png')
|
||
meta2 = wait_status(c, bob, att2['id'], ('ready',), timeout=120)
|
||
jid2, _, attempt2 = job_of(att2['id'])
|
||
for _ in range(60):
|
||
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid2) == '1':
|
||
break
|
||
time.sleep(0.5)
|
||
attempts = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid2)
|
||
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', compose('ps', '-q', 'relay').stdout.strip()).stdout.strip()
|
||
recreate('relay')
|
||
worker_log = compose('logs', '--no-color', 'worker').stdout
|
||
dup_seen = worker_log.count(jid2) >= 2
|
||
check(int(restarts) >= 1, 'fault relay.after_publish did not crash the relay')
|
||
check(attempt2 == 1, 'duplicate publish must not create a second attempt')
|
||
part2 = {'attachment_id': att2['id'], 'relay_restarts': int(restarts), 'outbox_publish_attempts': int(attempts or 0),
|
||
'worker_saw_message_twice': dup_seen, 'job_attempt': attempt2, 'invariants': invariants(att2['id'], chat['id'])}
|
||
return {'relay_down_then_up': part1, 'relay_crash_after_confirm': part2, 'events': events(jid2)}
|
||
|
||
|
||
def scenario_crash_effect(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
recreate('worker', WORKER_FAULT_POINTS='worker.after_side_effect', JOB_TIMEOUT_SECONDS='15')
|
||
t0 = time.time()
|
||
att = upload(c, alice, chat, sample(), name='crash.png')
|
||
meta = wait_status(c, bob, att['id'], ('ready',), timeout=120)
|
||
took = time.time() - t0
|
||
jid, state, attempt = job_of(att['id'])
|
||
cid = compose('ps', '-q', 'worker').stdout.split()[0]
|
||
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid).stdout.strip()
|
||
recreate('worker')
|
||
check(attempt == 2, 'expected recovery on attempt 2, got %d; events=%s' % (attempt, events(jid)))
|
||
ev = events(jid)
|
||
check('lease_expired_requeued' in ev, 'lease expiry recovery missing: ' + ev)
|
||
return {'attachment_id': att['id'], 'job_id': jid, 'worker_restarts': int(restarts), 'attempt': attempt,
|
||
'seconds_to_ready': round(took, 1), 'events': ev, 'invariants': invariants(att['id'], chat['id']),
|
||
'variants': check_variants(c, bob, meta, sample())}
|
||
|
||
|
||
def scenario_crash_commit(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
recreate('worker', WORKER_FAULT_POINTS='worker.after_commit')
|
||
att = upload(c, alice, chat, sample(), name='commit.png')
|
||
wait_status(c, bob, att['id'], ('ready',))
|
||
time.sleep(8) # контейнер перезапускается, broker доставляет неподтверждённое сообщение повторно
|
||
jid, state, attempt = job_of(att['id'])
|
||
cid = compose('ps', '-q', 'worker').stdout.split()[0]
|
||
restarts = sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid).stdout.strip()
|
||
log = compose('logs', '--no-color', 'worker').stdout
|
||
noop = any(jid in l and 'duplicate delivery of finished job' in l for l in log.splitlines())
|
||
recreate('worker')
|
||
check(int(restarts) >= 1, 'fault worker.after_commit did not crash the worker')
|
||
check(attempt == 1 and state == 'done', 'redelivery after commit must be a no-op: %s %d' % (state, attempt))
|
||
return {'attachment_id': att['id'], 'job_id': jid, 'worker_restarts': int(restarts), 'attempt': attempt,
|
||
'redelivery_acked_as_noop': noop, 'invariants': invariants(att['id'], chat['id'])}
|
||
|
||
|
||
def scenario_broker_restart(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
compose('stop', 'worker')
|
||
att = upload(c, alice, chat, sample(), name='broker.png')
|
||
jid, _, _ = job_of(att['id'])
|
||
for _ in range(30):
|
||
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
|
||
break
|
||
time.sleep(0.5)
|
||
before = queues()
|
||
compose('restart', 'rabbitmq')
|
||
# Quorum-очередь после рестарта выбирает лидера и восстанавливает журнал: ждём.
|
||
after = {}
|
||
for _ in range(30):
|
||
after = queues()
|
||
if after.get('attachment.process', 0) >= 1:
|
||
break
|
||
time.sleep(1)
|
||
sh('docker', 'start', compose('ps', '-aq', 'worker').stdout.split()[0]) # без запуска зависимостей
|
||
wait_status(c, bob, att['id'], ('ready',))
|
||
published = psql("SELECT publish_attempts FROM outbox WHERE job_id='%s'" % jid)
|
||
check(after.get('attachment.process', 0) >= 1, 'message lost on broker restart: %s' % after)
|
||
check(published == '1', 'message was republished from outbox (%s): not a broker durability proof' % published)
|
||
return {'attachment_id': att['id'], 'queue_before_restart': before, 'queue_after_restart': after,
|
||
'outbox_publish_attempts': int(published),
|
||
'invariants': invariants(att['id'], chat['id'])}
|
||
|
||
|
||
def scenario_transient(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
worker_cid = compose('ps', '-q', 'worker').stdout.split()[0]
|
||
s3_cid = compose('ps', '-q', 's3').stdout.split()[0]
|
||
compose('stop', 'worker')
|
||
att = upload(c, alice, chat, sample(), name='retry.png')
|
||
jid, _, _ = job_of(att['id'])
|
||
for _ in range(30):
|
||
if psql("SELECT count(*) FROM outbox WHERE job_id='%s' AND published_at IS NOT NULL" % jid) == '1':
|
||
break
|
||
time.sleep(0.5)
|
||
# Прямые docker stop/start: `docker compose start worker` поднял бы и S3
|
||
# (worker → s3-init → s3), и временного отказа бы не было.
|
||
sh('docker', 'stop', s3_cid)
|
||
sh('docker', 'start', worker_cid)
|
||
seen = []
|
||
t0 = time.time()
|
||
while time.time() - t0 < 25:
|
||
ev = events(jid) or ''
|
||
seen = ev.split(',')
|
||
if seen.count('retry_scheduled') >= 2:
|
||
break
|
||
time.sleep(1)
|
||
sh('docker', 'start', s3_cid)
|
||
meta = wait_status(c, bob, att['id'], ('ready',), timeout=150)
|
||
_, state, attempt = job_of(att['id'])
|
||
details = psql("SELECT string_agg(detail, ' | ' ORDER BY id) FROM job_events WHERE job_id='%s' AND event='retry_scheduled'" % jid)
|
||
check(attempt >= 2 and state == 'done', 'expected retries then success: %s/%d events=%s' % (state, attempt, events(jid)))
|
||
return {'attachment_id': att['id'], 'attempts': attempt, 'final_status': meta['status'],
|
||
'retry_details': details, 'events': events(jid), 'invariants': invariants(att['id'], chat['id'])}
|
||
|
||
|
||
def scenario_failed_replay(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
compose('stop', 'worker')
|
||
att = upload(c, alice, chat, sample(), name='bad.png')
|
||
key = psql("SELECT staging_key FROM attachments WHERE id='%s'" % att['id'])
|
||
mc('printf garbage > /tmp/g && mc cp -q /tmp/g s3/%s/%s >/dev/null' % (BUCKET, key)) # порча staging
|
||
compose('start', 'worker')
|
||
meta = wait_status(c, bob, att['id'], ('failed',))
|
||
check(meta['error_code'] == 'checksum_mismatch', 'error_code %s' % meta['error_code'])
|
||
c.call('GET', '/api/v1/attachments/%s/content' % att['id'], 409, actor=bob)
|
||
c.call('POST', '/api/v1/chats/%s/messages' % chat['id'], 409, actor=alice,
|
||
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': 'f-' + att['id']})
|
||
registry = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'failed').stdout
|
||
check(att['id'] in registry, 'failed registry must list the job')
|
||
# Восстанавливаем исходник и делаем явный replay.
|
||
import subprocess
|
||
p = subprocess.run(['docker', 'compose', 'run', '--rm', '--no-deps', '-T', '--entrypoint', 'sh', 's3-init', '-c',
|
||
'mc alias set s3 http://s3:9000 "$(cat /run/secrets/s3_root_user)" "$(cat /run/secrets/s3_root_password)" >/dev/null; '
|
||
'cat > /tmp/o && mc cp -q /tmp/o s3/%s/%s >/dev/null' % (BUCKET, key)], input=sample(), capture_output=True)
|
||
check(p.returncode == 0, 'restore staging failed')
|
||
rp = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'replay', att['id'],
|
||
'--actor', 'lab5-test', check=False)
|
||
check(rp.returncode == 0, 'replay failed: ' + rp.stdout + rp.stderr[-200:])
|
||
ready = wait_status(c, bob, att['id'], ('ready',))
|
||
again = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'replay', att['id'],
|
||
'--actor', 'lab5-test', check=False)
|
||
check(again.returncode != 0, 'replay of a finished job must be refused')
|
||
jid, _, attempt = job_of(att['id'])
|
||
# Невалидное сообщение → DLQ (не крутится вечно, не теряется молча).
|
||
psql("INSERT INTO outbox (job_id, payload) VALUES ('%s', '{\"schema_version\": 99}'::jsonb)" % jid)
|
||
dead = 0
|
||
for _ in range(30):
|
||
dead = queues().get('attachment.dead', 0)
|
||
if dead >= 1:
|
||
break
|
||
time.sleep(1)
|
||
check(dead >= 1, 'invalid message did not reach DLQ')
|
||
return {'attachment_id': att['id'], 'failed_error_code': meta['error_code'], 'content_while_failed': 409,
|
||
'attach_failed_to_message': 409, 'registry_listed': True, 'replay': json.loads(rp.stdout.strip().splitlines()[-1]),
|
||
'replay_after_success': json.loads(again.stdout.strip().splitlines()[-1]),
|
||
'final_status': ready['status'], 'events': events(jid), 'dlq_messages': dead,
|
||
'invariants': invariants(att['id'], chat['id'])}
|
||
|
||
|
||
def solid_png(w, h, rgb=(40, 120, 200)):
|
||
row = b'\x00' + bytes(rgb) * w
|
||
raw = row * h
|
||
ihdr = struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)
|
||
ch = lambda t, d: struct.pack('>I', len(d)) + t + d + struct.pack('>I', zlib.crc32(t + d) & 0xffffffff)
|
||
return b'\x89PNG\r\n\x1a\n' + ch(b'IHDR', ihdr) + ch(b'IDAT', zlib.compress(raw, 9)) + ch(b'IEND', b'')
|
||
|
||
|
||
def scenario_limits(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
big = solid_png(6000, 6000) # 36 Мпикс, сжатый файл маленький
|
||
check(len(big) < 10 * 1024 * 1024, 'test image too big')
|
||
import threading
|
||
compose('restart', 'worker') # новый cgroup → счётчики с нуля
|
||
time.sleep(3)
|
||
cid = compose('ps', '-q', 'worker').stdout.split()[0]
|
||
|
||
def cg(name):
|
||
return sh('docker', 'exec', cid, 'cat', '/sys/fs/cgroup/' + name, check=False).stdout
|
||
|
||
anon_max = [0]
|
||
stop = threading.Event()
|
||
|
||
def sampler():
|
||
# memory.peak включает page cache (временные файлы cwebp и т. п.), поэтому
|
||
# отдельно снимаем анонимную память процесса — её и ограничивает пиксельный бюджет.
|
||
while not stop.is_set():
|
||
for line in cg('memory.stat').splitlines():
|
||
if line.startswith('anon '):
|
||
anon_max[0] = max(anon_max[0], int(line.split()[1]))
|
||
time.sleep(0.3)
|
||
th = threading.Thread(target=sampler, daemon=True)
|
||
th.start()
|
||
t0 = time.time()
|
||
ids = [upload(c, alice, chat, big, name='big%d.png' % i)['id'] for i in range(3)]
|
||
done = [wait_status(c, bob, i, ('ready', 'failed'), timeout=240) for i in ids]
|
||
took = time.time() - t0
|
||
stop.set()
|
||
th.join()
|
||
peak = cg('memory.peak').strip()
|
||
limit = cg('memory.max').strip()
|
||
oom = {l.split()[0]: int(l.split()[1]) for l in cg('memory.events').splitlines() if len(l.split()) == 2}
|
||
check(all(d['status'] == 'ready' for d in done), 'big images failed: %s' % [d['error_code'] for d in done])
|
||
check(oom.get('oom_kill', 0) == 0, 'worker was OOM-killed')
|
||
th = {v['name']: (v['width'], v['height']) for v in done[0]['variants']}
|
||
check(th['thumbnail'] == (256, 256) and th['optimized'] == (1280, 1280), 'variant sizes %s' % th)
|
||
return {'image': '6000x6000 PNG (%d bytes)' % len(big), 'parallel_uploads': 3, 'seconds_all_ready': round(took, 1),
|
||
'worker_anon_memory_max_mib': round(anon_max[0] / 1048576, 1),
|
||
'worker_memory_peak_incl_page_cache_mib': round(int(peak) / 1048576, 1) if peak.isdigit() else peak,
|
||
'oom_kill_events': oom.get('oom_kill', 0),
|
||
'go_memlimit': sh('docker', 'exec', cid, 'printenv', 'GOMEMLIMIT', check=False).stdout.strip(),
|
||
'oom_events': oom.get('oom', 0),
|
||
'worker_memory_limit_mib': round(int(limit) / 1048576, 1) if limit.isdigit() else limit,
|
||
'pixel_budget': 60_000_000, 'decoded_rgba_per_image_mib': round(6000 * 6000 * 4 / 1048576, 1),
|
||
'variant_sizes': th}
|
||
|
||
|
||
# ---------------- трек 5: версии и устаревшие задания ----------------
|
||
|
||
def scenario_versions(c):
|
||
alice, bob, _, chat = c.bootstrap()
|
||
att = upload(c, alice, chat, sample(), name='ver.png')
|
||
v1 = wait_status(c, bob, att['id'], ('ready',))
|
||
th1 = {v['name']: v['sha256'] for v in v1['variants']}['thumbnail']
|
||
jid1, _, _ = job_of(att['id'], 1)
|
||
rp = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/pipeline', 'relay', 'reprocess', att['id'], '--version', '2')
|
||
jid2 = json.loads(rp.stdout.strip().splitlines()[-1])['job_id']
|
||
for _ in range(120):
|
||
if psql("SELECT pipeline_version FROM attachments WHERE id='%s'" % att['id']) == '2':
|
||
break
|
||
time.sleep(0.5)
|
||
v2 = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
|
||
th2 = {v['name']: v['sha256'] for v in v2['variants']}['thumbnail']
|
||
check(v2['status'] == 'ready' and th2 != th1, 'v2 must be published with a different thumbnail')
|
||
# Устаревший дубль v1 и повтор v2 после публикации v2.
|
||
psql("INSERT INTO outbox (job_id, payload) SELECT job_id, payload FROM outbox WHERE job_id='%s' ORDER BY id LIMIT 1" % jid1)
|
||
psql("INSERT INTO outbox (job_id, payload) SELECT job_id, payload FROM outbox WHERE job_id='%s' ORDER BY id LIMIT 1" % jid2)
|
||
time.sleep(6)
|
||
after = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']
|
||
th_after = {v['name']: v['sha256'] for v in after['variants']}['thumbnail']
|
||
got = c.call('GET', '/api/v1/attachments/%s/content?variant=thumbnail' % att['id'], actor=bob)['bytes']
|
||
check(th_after == th2 and hashlib.sha256(got).hexdigest() == th2, 'stale v1 overwrote v2')
|
||
check(psql("SELECT pipeline_version FROM attachments WHERE id='%s'" % att['id']) == '2', 'version went back')
|
||
_, s1, a1 = job_of(att['id'], 1)
|
||
_, s2, a2 = job_of(att['id'], 2)
|
||
check(a1 == 1 and a2 == 1, 'duplicates must not start new attempts')
|
||
return {'attachment_id': att['id'], 'v1_thumbnail_sha': th1, 'v2_thumbnail_sha': th2,
|
||
'after_stale_v1_and_dup_v2': th_after, 'published_version': 2,
|
||
'jobs': {'v1': {'state': s1, 'attempt': a1, 'events': events(jid1)},
|
||
'v2': {'state': s2, 'attempt': a2, 'events': events(jid2)}},
|
||
'invariants': invariants(att['id'], chat['id'])}
|
||
|
||
|
||
SCENARIOS = {'demo': scenario_demo, 'outbox-gap': scenario_outbox_gap, 'crash-effect': scenario_crash_effect,
|
||
'crash-commit': scenario_crash_commit, 'broker-restart': scenario_broker_restart,
|
||
'transient': scenario_transient, 'failed-replay': scenario_failed_replay,
|
||
'limits': scenario_limits, 'versions': scenario_versions}
|
||
|
||
|
||
def main():
|
||
def configure(p):
|
||
p.add_argument('scenario', choices=sorted(SCENARIOS))
|
||
p.add_argument('--output', default=None)
|
||
args = arguments('Lab5 pipeline scenarios on the Docker stand', configure)
|
||
c = Client(args.base_url, args.ca_file, timeout=60)
|
||
drain()
|
||
try:
|
||
result = SCENARIOS[args.scenario](c)
|
||
finally:
|
||
# Сценарий мог остановить сервисы или включить точки сбоя: возвращаем штатную конфигурацию.
|
||
compose('up', '-d', '--no-deps', '--wait', 'worker', 'relay', 'rabbitmq', 's3', 'api', timeout=300, check=False)
|
||
text = json.dumps({'scenario': args.scenario, 'result': result}, ensure_ascii=False, indent=2)
|
||
if args.output:
|
||
open(args.output, 'w', encoding='utf-8').write(text + '\n')
|
||
print(text)
|
||
print('PASS: ' + args.scenario)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|