forked from mtusi2026/mtusi2026lab5
429 lines
20 KiB
Python
429 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
sys.path.insert(0, str(ROOT / 'student-tests'))
|
|
sys.path.insert(0, str(ROOT / 'scripts'))
|
|
from client import Client, Failure, check
|
|
from image_probe import jpeg_size
|
|
import imagegen
|
|
from lablib import sh
|
|
|
|
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
NETWORK = 'mtusi-lab5_data'
|
|
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
|
|
|
|
|
|
def s3tool(*args):
|
|
cmd = ['docker', 'run', '--rm', '--network', NETWORK,
|
|
'-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro',
|
|
'-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro',
|
|
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
|
|
r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
|
|
if r.returncode != 0:
|
|
raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:])
|
|
return json.loads(r.stdout.strip().splitlines()[-1])
|
|
|
|
|
|
def psql(sql):
|
|
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
|
'-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql)
|
|
|
|
|
|
def queues():
|
|
out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues',
|
|
'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json')
|
|
return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'],
|
|
'consumers': q['consumers']} for q in json.loads(out)}
|
|
|
|
|
|
def restarts(service):
|
|
out = sh('docker', 'compose', 'ps', '-a', '-q', service)
|
|
return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()}
|
|
|
|
|
|
def events(att_id):
|
|
rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), "
|
|
"to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') "
|
|
"FROM job_events e JOIN attachment_jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
|
|
return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()]
|
|
|
|
|
|
def event_times(att_id):
|
|
rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) "
|
|
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
|
|
return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()]
|
|
|
|
|
|
def job_row(att_id):
|
|
r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), "
|
|
"(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), "
|
|
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), "
|
|
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), "
|
|
"a.staging_key IS NULL "
|
|
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id)
|
|
k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows',
|
|
'outbox_unpublished', 'staging_removed')
|
|
v = r.split('|')
|
|
d = dict(zip(k, v))
|
|
for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'):
|
|
d[f] = int(d[f])
|
|
d['staging_removed'] = d['staging_removed'] == 't'
|
|
return d
|
|
|
|
|
|
def s3_counts(chat_id, att_id):
|
|
return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'],
|
|
'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'],
|
|
'staging_objects': s3tool('count', 'staging/' + att_id)['count']}
|
|
|
|
|
|
def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202):
|
|
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
|
|
actor=actor, raw=data, headers={'Content-Type': ctype},
|
|
schema='Attachment' if expected == 202 else None)
|
|
|
|
|
|
def wait_status(c, actor, att_id, want, timeout=120):
|
|
deadline = time.monotonic() + timeout
|
|
seen = []
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
|
|
except Failure:
|
|
time.sleep(0.5)
|
|
continue
|
|
if not seen or seen[-1] != a['status']:
|
|
seen.append(a['status'])
|
|
if a['status'] in want:
|
|
return a, seen
|
|
time.sleep(0.3)
|
|
raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen))
|
|
|
|
|
|
def wait_api(c, timeout=90):
|
|
deadline = time.monotonic() + timeout
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Failure:
|
|
time.sleep(0.5)
|
|
raise Failure('API not ready')
|
|
|
|
|
|
def check_ready(c, actor, att):
|
|
names = sorted(v['name'] for v in att['variants'])
|
|
check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names)
|
|
orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)
|
|
check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed')
|
|
out = {'variants': {}}
|
|
for v in att['variants']:
|
|
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor)
|
|
check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum')
|
|
check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type')
|
|
out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes'])
|
|
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes']
|
|
out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb))
|
|
out['original_sha256_ok'] = True
|
|
return out
|
|
|
|
|
|
def fault_upload(c, actor, chat, point):
|
|
name = 'fault-%s-%d.png' % (point, int(time.time() * 1000))
|
|
try:
|
|
att = upload(c, actor, chat, name=name)['json']
|
|
return att['id'], name, 202
|
|
except Failure as exc:
|
|
att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name))
|
|
return att_id, name, 'client saw failure: ' + str(exc)[-60:]
|
|
|
|
|
|
def cmd_worker_down(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
steps = []
|
|
sh('docker', 'compose', 'stop', 'worker')
|
|
steps.append('docker compose stop worker')
|
|
t0 = time.monotonic()
|
|
r = upload(c, alice, chat)
|
|
att = r['json']
|
|
upload_ms = round((time.monotonic() - t0) * 1000)
|
|
steps.append('upload -> %d %s' % (r['status'], att['status']))
|
|
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
|
|
steps.append('message with queued attachment -> 201')
|
|
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
|
steps.append('content while queued -> 409')
|
|
c.message(bob, chat, 'текст работает без worker')
|
|
steps.append('plain text message -> 201')
|
|
time.sleep(2)
|
|
q_before = queues()
|
|
job_before = job_row(att['id'])
|
|
sh('docker', 'compose', 'restart', 'api')
|
|
sh('docker', 'compose', 'restart', 'rabbitmq')
|
|
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb')
|
|
wait_api(c)
|
|
steps.append('docker compose restart api rabbitmq (volumes kept)')
|
|
q_after = queues()
|
|
still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status']
|
|
sh('docker', 'compose', 'start', 'worker')
|
|
steps.append('docker compose start worker')
|
|
t1 = time.monotonic()
|
|
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
|
|
check(ready['status'] == 'ready', 'did not become ready')
|
|
res = {
|
|
'attachment_id': att['id'], 'message_id': msg['id'],
|
|
'upload_status': r['status'], 'upload_ms_worker_down': upload_ms,
|
|
'queue_before_restart': q_before.get('attachment.process.v1'),
|
|
'job_before_restart': job_before,
|
|
'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'),
|
|
'status_after_restart_before_worker': still,
|
|
'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2),
|
|
'statuses_seen': seen, 'steps': steps,
|
|
'ready_check': check_ready(c, bob, ready),
|
|
'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']),
|
|
}
|
|
return res
|
|
|
|
|
|
def cmd_api_crash(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
before = restarts('api')
|
|
att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit')
|
|
check(att_id, 'attachment row not found')
|
|
row = job_row(att_id)
|
|
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
|
check(ready['status'] == 'ready', 'not ready')
|
|
time.sleep(3)
|
|
return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response',
|
|
'attachment_id': att_id, 'client_view': client_view,
|
|
'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'),
|
|
'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id),
|
|
'ready_check': check_ready(c, bob, ready)}
|
|
|
|
|
|
def _worker_crash(a, point, desc):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
before = restarts('worker')
|
|
att_id, name, client_view = fault_upload(c, alice, chat, point)
|
|
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
|
check(ready['status'] == 'ready', 'not ready')
|
|
time.sleep(5)
|
|
job = job_row(att_id)
|
|
s3 = s3_counts(chat['id'], att_id)
|
|
check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants')
|
|
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m')
|
|
noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l]
|
|
return {'fault': desc, 'attachment_id': att_id, 'upload': client_view,
|
|
'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'),
|
|
'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3,
|
|
'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop],
|
|
'ready_check': check_ready(c, bob, ready)}
|
|
|
|
|
|
def cmd_effect_crash(a):
|
|
return _worker_crash(a, 'worker-after-effect',
|
|
'worker os.Exit after writing original+variants to S3, before result COMMIT and ack')
|
|
|
|
|
|
def cmd_ack_crash(a):
|
|
return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack')
|
|
|
|
|
|
def cmd_retry(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
att_id, _, _ = fault_upload(c, alice, chat, 'transient')
|
|
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
|
|
check(ready['status'] == 'ready', 'not ready')
|
|
t = event_times(att_id)
|
|
fails = [x for x in t if x[0] == 'retry_scheduled']
|
|
starts = [x for x in t if x[0] == 'started']
|
|
gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))]
|
|
return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)',
|
|
'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id),
|
|
'retry_gaps_seconds': gaps,
|
|
'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s',
|
|
'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'],
|
|
'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)}
|
|
|
|
|
|
def cmd_replay(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
att_id, _, _ = fault_upload(c, alice, chat, 'flaky')
|
|
failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180)
|
|
check(failed['status'] == 'failed', 'expected failed')
|
|
res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id,
|
|
'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')},
|
|
'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'],
|
|
'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
|
body={'text': 'x', 'attachment_ids': [att_id]},
|
|
headers={'Idempotency-Key': 'replay-' + att_id})['status'],
|
|
'job_failed': job_row(att_id)}
|
|
res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id)
|
|
replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id,
|
|
'--actor', 'lab5ops'))
|
|
res['replay_1'] = replay()
|
|
ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed'))
|
|
check(ready['status'] == 'ready', 'replay did not finish')
|
|
res['replay_2_after_ready'] = replay()
|
|
res['statuses_after_replay'] = seen2
|
|
res['events'] = events(att_id)
|
|
res['job_after'] = job_row(att_id)
|
|
res['ready_check'] = check_ready(c, bob, ready)
|
|
return res
|
|
|
|
|
|
def corrupt_png():
|
|
good = imagegen.png(64, 64)
|
|
return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) +
|
|
imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b''))
|
|
|
|
|
|
def cmd_invalid(a):
|
|
c = Client(a.base_url, timeout=60)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json']
|
|
failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
|
|
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image')
|
|
return {
|
|
'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'],
|
|
'events': events(att['id']), 'job': job_row(att['id'])},
|
|
'api_rejects_before_queue': {
|
|
'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'],
|
|
'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'],
|
|
'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1),
|
|
name='big.png', expected=413)['status'],
|
|
'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'],
|
|
}}
|
|
|
|
|
|
def cmd_hang(a):
|
|
c = Client(a.base_url, timeout=30)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
att_id, _, _ = fault_upload(c, alice, chat, 'hang')
|
|
t0 = time.monotonic()
|
|
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150)
|
|
check(ready['status'] == 'ready', 'not ready')
|
|
took = round(time.monotonic() - t0, 1)
|
|
time.sleep(70)
|
|
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m')
|
|
stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)]
|
|
return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)',
|
|
'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen,
|
|
'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id),
|
|
'stale_attempt_logs': [json.loads(l).get('msg') for l in stale],
|
|
'ready_check': check_ready(c, bob, ready)}
|
|
|
|
|
|
def cmd_dlq(a):
|
|
before = queues()
|
|
script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
|
|
'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
|
|
'wget -q -O- --header "Content-Type: application/json" --post-data '
|
|
'\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",'
|
|
'"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' '
|
|
'"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"')
|
|
routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script)
|
|
time.sleep(3)
|
|
after = queues()
|
|
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m')
|
|
return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'),
|
|
'dlq_after': after.get('attachment.process.dlq'),
|
|
'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]}
|
|
|
|
|
|
def gradient_png(w, h):
|
|
row = b'\x02' + bytes(3 * w)
|
|
first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3))
|
|
raw = first + row * (h - 1)
|
|
return imagegen.png(w, h, raw=raw, level=9)
|
|
|
|
|
|
class MemSampler:
|
|
def __init__(self, service):
|
|
self.cids = sh('docker', 'compose', 'ps', '-q', service).split()
|
|
self.peak = 0.0
|
|
self._stop = threading.Event()
|
|
self.t = threading.Thread(target=self.run, daemon=True)
|
|
|
|
def run(self):
|
|
while not self._stop.is_set():
|
|
out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False)
|
|
for line in out.splitlines():
|
|
used = line.split('/')[0].strip()
|
|
num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0)
|
|
mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num
|
|
self.peak = max(self.peak, mib)
|
|
|
|
def __enter__(self):
|
|
self.t.start()
|
|
return self
|
|
|
|
def __exit__(self, *exc):
|
|
self._stop.set()
|
|
self.t.join(timeout=10)
|
|
|
|
|
|
def cmd_limits(a):
|
|
c = Client(a.base_url, timeout=60)
|
|
alice, bob, _eve, chat = c.bootstrap()
|
|
data = gradient_png(6000, 6000)
|
|
limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0])
|
|
with MemSampler('worker') as m:
|
|
t0 = time.monotonic()
|
|
att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json']
|
|
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120)
|
|
took = round(time.monotonic() - t0, 1)
|
|
time.sleep(2)
|
|
check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code'])
|
|
return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took,
|
|
'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20),
|
|
'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']},
|
|
'statuses_seen': seen, 'job': job_row(att['id'])}
|
|
|
|
|
|
COMMANDS = {
|
|
'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash,
|
|
'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid,
|
|
'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
p.add_argument('command', choices=sorted(COMMANDS))
|
|
p.add_argument('--base-url', default='http://localhost:8080')
|
|
p.add_argument('--output')
|
|
a = p.parse_args()
|
|
started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
|
try:
|
|
res = COMMANDS[a.command](a)
|
|
res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res}
|
|
except (Failure, RuntimeError) as exc:
|
|
res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)}
|
|
text = json.dumps(res, ensure_ascii=False, indent=2)
|
|
print(text)
|
|
if a.output:
|
|
Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
|
|
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
|
|
if res['result'] != 'PASS':
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|