Files
mtusi2026lab5/scripts/lab5ops.py

429 lines
20 KiB
Python

#!/usr/bin/env python3
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
sys.path.insert(0, str(ROOT / 'scripts'))
from client import Client, Failure, check
from image_probe import jpeg_size
import imagegen
from lablib import sh
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
NETWORK = 'mtusi-lab5_data'
TOOL_IMAGE = 'python:3.12.8-alpine3.21'
def s3tool(*args):
cmd = ['docker', 'run', '--rm', '--network', NETWORK,
'-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro',
'-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro',
TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args]
r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8')
if r.returncode != 0:
raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:])
return json.loads(r.stdout.strip().splitlines()[-1])
def psql(sql):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql)
def queues():
out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues',
'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json')
return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'],
'consumers': q['consumers']} for q in json.loads(out)}
def restarts(service):
out = sh('docker', 'compose', 'ps', '-a', '-q', service)
return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()}
def events(att_id):
rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), "
"to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') "
"FROM job_events e JOIN attachment_jobs j USING (job_id) "
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()]
def event_times(att_id):
rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) "
"WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id)
return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()]
def job_row(att_id):
r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), "
"(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), "
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), "
"(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), "
"a.staging_key IS NULL "
"FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id)
k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows',
'outbox_unpublished', 'staging_removed')
v = r.split('|')
d = dict(zip(k, v))
for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'):
d[f] = int(d[f])
d['staging_removed'] = d['staging_removed'] == 't'
return d
def s3_counts(chat_id, att_id):
return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'],
'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'],
'staging_objects': s3tool('count', 'staging/' + att_id)['count']}
def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected,
actor=actor, raw=data, headers={'Content-Type': ctype},
schema='Attachment' if expected == 202 else None)
def wait_status(c, actor, att_id, want, timeout=120):
deadline = time.monotonic() + timeout
seen = []
while time.monotonic() < deadline:
try:
a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
except Failure:
time.sleep(0.5)
continue
if not seen or seen[-1] != a['status']:
seen.append(a['status'])
if a['status'] in want:
return a, seen
time.sleep(0.3)
raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen))
def wait_api(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise Failure('API not ready')
def check_ready(c, actor, att):
names = sorted(v['name'] for v in att['variants'])
check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names)
orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor)
check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed')
out = {'variants': {}}
for v in att['variants']:
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor)
check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum')
check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type')
out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes'])
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes']
out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb))
out['original_sha256_ok'] = True
return out
def fault_upload(c, actor, chat, point):
name = 'fault-%s-%d.png' % (point, int(time.time() * 1000))
try:
att = upload(c, actor, chat, name=name)['json']
return att['id'], name, 202
except Failure as exc:
att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name))
return att_id, name, 'client saw failure: ' + str(exc)[-60:]
def cmd_worker_down(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
steps = []
sh('docker', 'compose', 'stop', 'worker')
steps.append('docker compose stop worker')
t0 = time.monotonic()
r = upload(c, alice, chat)
att = r['json']
upload_ms = round((time.monotonic() - t0) * 1000)
steps.append('upload -> %d %s' % (r['status'], att['status']))
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
steps.append('message with queued attachment -> 201')
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
steps.append('content while queued -> 409')
c.message(bob, chat, 'текст работает без worker')
steps.append('plain text message -> 201')
time.sleep(2)
q_before = queues()
job_before = job_row(att['id'])
sh('docker', 'compose', 'restart', 'api')
sh('docker', 'compose', 'restart', 'rabbitmq')
sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb')
wait_api(c)
steps.append('docker compose restart api rabbitmq (volumes kept)')
q_after = queues()
still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status']
sh('docker', 'compose', 'start', 'worker')
steps.append('docker compose start worker')
t1 = time.monotonic()
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
check(ready['status'] == 'ready', 'did not become ready')
res = {
'attachment_id': att['id'], 'message_id': msg['id'],
'upload_status': r['status'], 'upload_ms_worker_down': upload_ms,
'queue_before_restart': q_before.get('attachment.process.v1'),
'job_before_restart': job_before,
'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'),
'status_after_restart_before_worker': still,
'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2),
'statuses_seen': seen, 'steps': steps,
'ready_check': check_ready(c, bob, ready),
'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']),
}
return res
def cmd_api_crash(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
before = restarts('api')
att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit')
check(att_id, 'attachment row not found')
row = job_row(att_id)
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
time.sleep(3)
return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response',
'attachment_id': att_id, 'client_view': client_view,
'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'),
'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id),
'ready_check': check_ready(c, bob, ready)}
def _worker_crash(a, point, desc):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
before = restarts('worker')
att_id, name, client_view = fault_upload(c, alice, chat, point)
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
time.sleep(5)
job = job_row(att_id)
s3 = s3_counts(chat['id'], att_id)
check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants')
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m')
noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l]
return {'fault': desc, 'attachment_id': att_id, 'upload': client_view,
'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'),
'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3,
'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop],
'ready_check': check_ready(c, bob, ready)}
def cmd_effect_crash(a):
return _worker_crash(a, 'worker-after-effect',
'worker os.Exit after writing original+variants to S3, before result COMMIT and ack')
def cmd_ack_crash(a):
return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack')
def cmd_retry(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'transient')
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'not ready')
t = event_times(att_id)
fails = [x for x in t if x[0] == 'retry_scheduled']
starts = [x for x in t if x[0] == 'started']
gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))]
return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)',
'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id),
'retry_gaps_seconds': gaps,
'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s',
'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'],
'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)}
def cmd_replay(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'flaky')
failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180)
check(failed['status'] == 'failed', 'expected failed')
res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id,
'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')},
'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'],
'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
body={'text': 'x', 'attachment_ids': [att_id]},
headers={'Idempotency-Key': 'replay-' + att_id})['status'],
'job_failed': job_row(att_id)}
res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id)
replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id,
'--actor', 'lab5ops'))
res['replay_1'] = replay()
ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed'))
check(ready['status'] == 'ready', 'replay did not finish')
res['replay_2_after_ready'] = replay()
res['statuses_after_replay'] = seen2
res['events'] = events(att_id)
res['job_after'] = job_row(att_id)
res['ready_check'] = check_ready(c, bob, ready)
return res
def corrupt_png():
good = imagegen.png(64, 64)
return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) +
imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b''))
def cmd_invalid(a):
c = Client(a.base_url, timeout=60)
alice, bob, _eve, chat = c.bootstrap()
att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json']
failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed'))
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image')
return {
'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'],
'events': events(att['id']), 'job': job_row(att['id'])},
'api_rejects_before_queue': {
'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'],
'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'],
'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1),
name='big.png', expected=413)['status'],
'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'],
}}
def cmd_hang(a):
c = Client(a.base_url, timeout=30)
alice, bob, _eve, chat = c.bootstrap()
att_id, _, _ = fault_upload(c, alice, chat, 'hang')
t0 = time.monotonic()
ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150)
check(ready['status'] == 'ready', 'not ready')
took = round(time.monotonic() - t0, 1)
time.sleep(70)
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m')
stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)]
return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)',
'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen,
'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id),
'stale_attempt_logs': [json.loads(l).get('msg') for l in stale],
'ready_check': check_ready(c, bob, ready)}
def cmd_dlq(a):
before = queues()
script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); '
'wget -q -O- --header "Content-Type: application/json" --post-data '
'\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",'
'"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' '
'"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"')
routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script)
time.sleep(3)
after = queues()
logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m')
return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'),
'dlq_after': after.get('attachment.process.dlq'),
'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]}
def gradient_png(w, h):
row = b'\x02' + bytes(3 * w)
first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3))
raw = first + row * (h - 1)
return imagegen.png(w, h, raw=raw, level=9)
class MemSampler:
def __init__(self, service):
self.cids = sh('docker', 'compose', 'ps', '-q', service).split()
self.peak = 0.0
self._stop = threading.Event()
self.t = threading.Thread(target=self.run, daemon=True)
def run(self):
while not self._stop.is_set():
out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False)
for line in out.splitlines():
used = line.split('/')[0].strip()
num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0)
mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num
self.peak = max(self.peak, mib)
def __enter__(self):
self.t.start()
return self
def __exit__(self, *exc):
self._stop.set()
self.t.join(timeout=10)
def cmd_limits(a):
c = Client(a.base_url, timeout=60)
alice, bob, _eve, chat = c.bootstrap()
data = gradient_png(6000, 6000)
limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0])
with MemSampler('worker') as m:
t0 = time.monotonic()
att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json']
ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120)
took = round(time.monotonic() - t0, 1)
time.sleep(2)
check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code'])
return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took,
'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20),
'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']},
'statuses_seen': seen, 'job': job_row(att['id'])}
COMMANDS = {
'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash,
'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid,
'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits,
}
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument('command', choices=sorted(COMMANDS))
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output')
a = p.parse_args()
started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
try:
res = COMMANDS[a.command](a)
res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res}
except (Failure, RuntimeError) as exc:
res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)}
text = json.dumps(res, ensure_ascii=False, indent=2)
print(text)
if a.output:
Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True)
(ROOT / a.output).write_text(text + '\n', encoding='utf-8')
if res['result'] != 'PASS':
raise SystemExit(1)
if __name__ == '__main__':
main()