#!/usr/bin/env python3 import argparse import hashlib import json from pathlib import Path import subprocess import sys import threading import time import zlib ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) sys.path.insert(0, str(ROOT / 'student-tests')) sys.path.insert(0, str(ROOT / 'scripts')) from client import Client, Failure, check from image_probe import jpeg_size import imagegen from lablib import sh PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes() NETWORK = 'mtusi-lab5_data' TOOL_IMAGE = 'python:3.12.8-alpine3.21' def s3tool(*args): cmd = ['docker', 'run', '--rm', '--network', NETWORK, '-v', str(ROOT / 'secrets').replace('\\', '/') + ':/secrets:ro', '-v', str(ROOT / 'scripts').replace('\\', '/') + ':/tool:ro', TOOL_IMAGE, 'python', '-I', '/tool/s3tool.py', *args] r = subprocess.run(cmd, capture_output=True, text=True, encoding='utf-8') if r.returncode != 0: raise RuntimeError('s3tool failed: ' + (r.stdout + r.stderr).strip()[-300:]) return json.loads(r.stdout.strip().splitlines()[-1]) def psql(sql): return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-At', '-F', '|', '-v', 'ON_ERROR_STOP=1', '-c', sql) def queues(): out = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'messages_ready', 'messages_unacknowledged', 'consumers', '--formatter', 'json') return {q['name']: {'ready': q['messages_ready'], 'unacked': q['messages_unacknowledged'], 'consumers': q['consumers']} for q in json.loads(out)} def restarts(service): out = sh('docker', 'compose', 'ps', '-a', '-q', service) return {cid[:12]: int(sh('docker', 'inspect', '-f', '{{.RestartCount}}', cid)) for cid in out.split()} def events(att_id): rows = psql("SELECT e.attempt, e.event, COALESCE(e.error_code,''), " "to_char(e.created_at AT TIME ZONE 'UTC','HH24:MI:SS.MS') " "FROM job_events e JOIN attachment_jobs j USING (job_id) " "WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id) return [dict(zip(('attempt', 'event', 'error_code', 'at'), r.split('|'))) for r in rows.splitlines()] def event_times(att_id): rows = psql("SELECT e.event, extract(epoch FROM e.created_at) FROM job_events e JOIN attachment_jobs j USING (job_id) " "WHERE j.attachment_id = '%s' ORDER BY e.id" % att_id) return [(r.split('|')[0], float(r.split('|')[1])) for r in rows.splitlines()] def job_row(att_id): r = psql("SELECT j.job_id, j.status, j.attempt, j.max_attempts, COALESCE(j.error_code,''), " "(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id), " "(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id), " "(SELECT count(*) FROM job_outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL), " "a.staging_key IS NULL " "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.attachment_id = '%s'" % att_id) k = ('job_id', 'status', 'attempt', 'max_attempts', 'error_code', 'variant_rows', 'outbox_rows', 'outbox_unpublished', 'staging_removed') v = r.split('|') d = dict(zip(k, v)) for f in ('attempt', 'max_attempts', 'variant_rows', 'outbox_rows', 'outbox_unpublished'): d[f] = int(d[f]) d['staging_removed'] = d['staging_removed'] == 't' return d def s3_counts(chat_id, att_id): return {'variants_objects': s3tool('count', 'variants/%s/%s/' % (chat_id, att_id))['count'], 'final_original': s3tool('count', 'attachments/%s/%s' % (chat_id, att_id))['count'], 'staging_objects': s3tool('count', 'staging/' + att_id)['count']} def upload(c, actor, chat, data=PNG, name='sample.png', ctype='image/png', expected=202): return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + name, expected, actor=actor, raw=data, headers={'Content-Type': ctype}, schema='Attachment' if expected == 202 else None) def wait_status(c, actor, att_id, want, timeout=120): deadline = time.monotonic() + timeout seen = [] while time.monotonic() < deadline: try: a = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] except Failure: time.sleep(0.5) continue if not seen or seen[-1] != a['status']: seen.append(a['status']) if a['status'] in want: return a, seen time.sleep(0.3) raise Failure('attachment did not reach %s in %ss (seen %s)' % (want, timeout, seen)) def wait_api(c, timeout=90): deadline = time.monotonic() + timeout while time.monotonic() < deadline: try: c.call('GET', '/health/ready', schema='Health') return except Failure: time.sleep(0.5) raise Failure('API not ready') def check_ready(c, actor, att): names = sorted(v['name'] for v in att['variants']) check(names == ['crop', 'optimized', 'thumbnail'], 'variants: %s' % names) orig = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=actor) check(hashlib.sha256(orig['bytes']).hexdigest() == att['sha256'], 'original changed') out = {'variants': {}} for v in att['variants']: r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=actor) check(hashlib.sha256(r['bytes']).hexdigest() == v['sha256'], v['name'] + ' checksum') check(r['headers'].get_content_type() == v['content_type'], v['name'] + ' content type') out['variants'][v['name']] = '%s %dx%d %dB' % (v['content_type'], v['width'], v['height'], v['size_bytes']) thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=actor)['bytes'] out['thumbnail_real_jpeg_size'] = list(jpeg_size(thumb)) out['original_sha256_ok'] = True return out def fault_upload(c, actor, chat, point): name = 'fault-%s-%d.png' % (point, int(time.time() * 1000)) try: att = upload(c, actor, chat, name=name)['json'] return att['id'], name, 202 except Failure as exc: att_id = psql("SELECT id FROM attachments WHERE chat_id = '%s' AND filename = '%s'" % (chat['id'], name)) return att_id, name, 'client saw failure: ' + str(exc)[-60:] def cmd_worker_down(a): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() steps = [] sh('docker', 'compose', 'stop', 'worker') steps.append('docker compose stop worker') t0 = time.monotonic() r = upload(c, alice, chat) att = r['json'] upload_ms = round((time.monotonic() - t0) * 1000) steps.append('upload -> %d %s' % (r['status'], att['status'])) msg = c.message(alice, chat, '', attachments=[att['id']])['json'] steps.append('message with queued attachment -> 201') c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) steps.append('content while queued -> 409') c.message(bob, chat, 'текст работает без worker') steps.append('plain text message -> 201') time.sleep(2) q_before = queues() job_before = job_row(att['id']) sh('docker', 'compose', 'restart', 'api') sh('docker', 'compose', 'restart', 'rabbitmq') sh('docker', 'compose', 'up', '-d', '--no-recreate', '--wait', 'rabbitmq', 'api', 'lb') wait_api(c) steps.append('docker compose restart api rabbitmq (volumes kept)') q_after = queues() still = c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] sh('docker', 'compose', 'start', 'worker') steps.append('docker compose start worker') t1 = time.monotonic() ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed')) check(ready['status'] == 'ready', 'did not become ready') res = { 'attachment_id': att['id'], 'message_id': msg['id'], 'upload_status': r['status'], 'upload_ms_worker_down': upload_ms, 'queue_before_restart': q_before.get('attachment.process.v1'), 'job_before_restart': job_before, 'queue_after_api_and_broker_restart': q_after.get('attachment.process.v1'), 'status_after_restart_before_worker': still, 'seconds_to_ready_after_worker_start': round(time.monotonic() - t1, 2), 'statuses_seen': seen, 'steps': steps, 'ready_check': check_ready(c, bob, ready), 'job_after': job_row(att['id']), 's3': s3_counts(chat['id'], att['id']), } return res def cmd_api_crash(a): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() before = restarts('api') att_id, name, client_view = fault_upload(c, alice, chat, 'api-after-commit') check(att_id, 'attachment row not found') row = job_row(att_id) ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) check(ready['status'] == 'ready', 'not ready') time.sleep(3) return {'fault': 'API os.Exit right after COMMIT(queued+job+outbox), before publish and before HTTP response', 'attachment_id': att_id, 'client_view': client_view, 'job_right_after_crash': row, 'api_restart_counts_before': before, 'api_restart_counts_after': restarts('api'), 'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)} def _worker_crash(a, point, desc): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() before = restarts('worker') att_id, name, client_view = fault_upload(c, alice, chat, point) ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) check(ready['status'] == 'ready', 'not ready') time.sleep(5) job = job_row(att_id) s3 = s3_counts(chat['id'], att_id) check(job['variant_rows'] == 3 and s3['variants_objects'] == 3, 'duplicated or missing variants') logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '3m') noop = [json.loads(l) for l in logs.splitlines() if att_id in l and 'no-op' in l] return {'fault': desc, 'attachment_id': att_id, 'upload': client_view, 'worker_restart_counts_before': before, 'worker_restart_counts_after': restarts('worker'), 'statuses_seen': seen, 'events': events(att_id), 'job_after': job, 's3': s3, 'redelivery_noop_log': [{'result': n.get('result'), 'redelivered': n.get('redelivered')} for n in noop], 'ready_check': check_ready(c, bob, ready)} def cmd_effect_crash(a): return _worker_crash(a, 'worker-after-effect', 'worker os.Exit after writing original+variants to S3, before result COMMIT and ack') def cmd_ack_crash(a): return _worker_crash(a, 'worker-before-ack', 'worker os.Exit after result COMMIT + staging cleanup, before ack') def cmd_retry(a): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() att_id, _, _ = fault_upload(c, alice, chat, 'transient') ready, seen = wait_status(c, bob, att_id, ('ready', 'failed')) check(ready['status'] == 'ready', 'not ready') t = event_times(att_id) fails = [x for x in t if x[0] == 'retry_scheduled'] starts = [x for x in t if x[0] == 'started'] gaps = [round(starts[i + 1][1] - fails[i][1], 2) for i in range(len(fails))] return {'fault': 'attempts 1 and 2 raise a transient error (injected_transient)', 'attachment_id': att_id, 'statuses_seen': seen, 'events': events(att_id), 'retry_gaps_seconds': gaps, 'expected_gap_ranges': ['attempt1: backoff [0.5, 1.0] s + relay poll <= 1 s', 'attempt2: backoff [1.0, 2.0] s + relay poll <= 1 s'], 'job_after': job_row(att_id), 'ready_check': check_ready(c, bob, ready)} def cmd_replay(a): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() att_id, _, _ = fault_upload(c, alice, chat, 'flaky') failed, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=180) check(failed['status'] == 'failed', 'expected failed') res = {'fault': 'transient error on every attempt <= JOB_MAX_ATTEMPTS', 'attachment_id': att_id, 'statuses_seen': seen, 'failed_metadata': {k: failed[k] for k in ('status', 'error_code', 'variants')}, 'content_when_failed': c.call('GET', '/api/v1/attachments/' + att_id + '/content', 409, actor=bob)['status'], 'attach_failed_to_message': c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': 'x', 'attachment_ids': [att_id]}, headers={'Idempotency-Key': 'replay-' + att_id})['status'], 'job_failed': job_row(att_id)} res['dlq_registry_query'] = psql("SELECT count(*) FROM attachment_jobs WHERE status = 'failed' AND attachment_id = '%s'" % att_id) replay = lambda: json.loads(sh('docker', 'compose', 'exec', '-T', 'worker', '/app/server', 'replay', att_id, '--actor', 'lab5ops')) res['replay_1'] = replay() ready, seen2 = wait_status(c, bob, att_id, ('ready', 'failed')) check(ready['status'] == 'ready', 'replay did not finish') res['replay_2_after_ready'] = replay() res['statuses_after_replay'] = seen2 res['events'] = events(att_id) res['job_after'] = job_row(att_id) res['ready_check'] = check_ready(c, bob, ready) return res def corrupt_png(): good = imagegen.png(64, 64) return (b'\x89PNG\r\n\x1a\n' + imagegen.chunk(b'IHDR', good[16:29]) + imagegen.chunk(b'IDAT', b'not-a-zlib-stream' * 10) + imagegen.chunk(b'IEND', b'')) def cmd_invalid(a): c = Client(a.base_url, timeout=60) alice, bob, _eve, chat = c.bootstrap() att = upload(c, alice, chat, data=corrupt_png(), name='corrupt.png')['json'] failed, seen = wait_status(c, bob, att['id'], ('ready', 'failed')) check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image', 'expected invalid_image') return { 'corrupt_png': {'upload': 202, 'statuses_seen': seen, 'error_code': failed['error_code'], 'events': events(att['id']), 'job': job_row(att['id'])}, 'api_rejects_before_queue': { 'pixel_bomb_20000x20000': upload(c, alice, chat, data=imagegen.bomb_png(), name='bomb.png', expected=413)['status'], 'not_an_image': upload(c, alice, chat, data=b'GIF89a' + b'\0' * 100, name='x.png', expected=415)['status'], 'over_10MiB': upload(c, alice, chat, data=b'\x89PNG\r\n\x1a\n' + b'\0' * (10 * 1024 * 1024 + 1), name='big.png', expected=413)['status'], 'path_in_filename': upload(c, alice, chat, name='..%2Fetc.png', expected=400)['status'], }} def cmd_hang(a): c = Client(a.base_url, timeout=30) alice, bob, _eve, chat = c.bootstrap() att_id, _, _ = fault_upload(c, alice, chat, 'hang') t0 = time.monotonic() ready, seen = wait_status(c, bob, att_id, ('ready', 'failed'), timeout=150) check(ready['status'] == 'ready', 'not ready') took = round(time.monotonic() - t0, 1) time.sleep(70) logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '5m') stale = [l for l in logs.splitlines() if att_id in l and ('stale attempt' in l or 'no-op' in l)] return {'fault': 'attempt 1 freezes for 3x lease without heartbeat (simulated hung worker)', 'attachment_id': att_id, 'seconds_to_ready': took, 'statuses_seen': seen, 'events': events(att_id), 'job_after': job_row(att_id), 's3': s3_counts(chat['id'], att_id), 'stale_attempt_logs': [json.loads(l).get('msg') for l in stale], 'ready_check': check_ready(c, bob, ready)} def cmd_dlq(a): before = queues() script = ('U=$(sed -n "s/^default_user = //p" /etc/rabbitmq/conf.d/20-auth.conf); ' 'P=$(sed -n "s/^default_pass = //p" /etc/rabbitmq/conf.d/20-auth.conf); ' 'wget -q -O- --header "Content-Type: application/json" --post-data ' '\'{"properties":{"delivery_mode":2},"routing_key":"attachment.process.v1",' '"payload":"{\\"not\\":\\"a job\\"}","payload_encoding":"string"}\' ' '"http://$U:$P@127.0.0.1:15672/api/exchanges/%2F/attachments/publish"') routed = sh('docker', 'compose', 'exec', '-T', 'rabbitmq', 'sh', '-c', script) time.sleep(3) after = queues() logs = sh('docker', 'compose', 'logs', 'worker', '--no-log-prefix', '--since', '1m') return {'published': json.loads(routed), 'dlq_before': before.get('attachment.process.dlq'), 'dlq_after': after.get('attachment.process.dlq'), 'worker_log': [json.loads(l)['msg'] for l in logs.splitlines() if 'DLQ' in l][-1:]} def gradient_png(w, h): row = b'\x02' + bytes(3 * w) first = b'\x00' + bytes((x * 255 // max(1, w - 1)) for x in range(w) for _ in range(3)) raw = first + row * (h - 1) return imagegen.png(w, h, raw=raw, level=9) class MemSampler: def __init__(self, service): self.cids = sh('docker', 'compose', 'ps', '-q', service).split() self.peak = 0.0 self._stop = threading.Event() self.t = threading.Thread(target=self.run, daemon=True) def run(self): while not self._stop.is_set(): out = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *self.cids, check=False) for line in out.splitlines(): used = line.split('/')[0].strip() num = float(''.join(ch for ch in used if ch.isdigit() or ch == '.') or 0) mib = num * 1024 if used.endswith('GiB') else num / 1024 if used.endswith('KiB') else num self.peak = max(self.peak, mib) def __enter__(self): self.t.start() return self def __exit__(self, *exc): self._stop.set() self.t.join(timeout=10) def cmd_limits(a): c = Client(a.base_url, timeout=60) alice, bob, _eve, chat = c.bootstrap() data = gradient_png(6000, 6000) limit = sh('docker', 'inspect', '-f', '{{.HostConfig.Memory}}', sh('docker', 'compose', 'ps', '-q', 'worker').split()[0]) with MemSampler('worker') as m: t0 = time.monotonic() att = upload(c, alice, chat, data=data, name='big-6000x6000.png')['json'] ready, seen = wait_status(c, bob, att['id'], ('ready', 'failed'), timeout=120) took = round(time.monotonic() - t0, 1) time.sleep(2) check(ready['status'] == 'ready', 'big image failed: %s' % ready['error_code']) return {'image': '6000x6000 PNG (36 MPx, %d bytes)' % len(data), 'seconds_to_ready': took, 'worker_peak_mem_mib': round(m.peak, 1), 'worker_mem_limit_mib': int(limit) // (1 << 20), 'variants': {v['name']: '%dx%d' % (v['width'], v['height']) for v in ready['variants']}, 'statuses_seen': seen, 'job': job_row(att['id'])} COMMANDS = { 'worker-down': cmd_worker_down, 'api-crash': cmd_api_crash, 'effect-crash': cmd_effect_crash, 'ack-crash': cmd_ack_crash, 'retry': cmd_retry, 'replay': cmd_replay, 'invalid': cmd_invalid, 'hang': cmd_hang, 'dlq': cmd_dlq, 'limits': cmd_limits, } def main(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument('command', choices=sorted(COMMANDS)) p.add_argument('--base-url', default='http://localhost:8080') p.add_argument('--output') a = p.parse_args() started = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) try: res = COMMANDS[a.command](a) res = {'scenario': a.command, 'result': 'PASS', 'started_at': started, **res} except (Failure, RuntimeError) as exc: res = {'scenario': a.command, 'result': 'FAIL', 'started_at': started, 'error': str(exc)} text = json.dumps(res, ensure_ascii=False, indent=2) print(text) if a.output: Path(ROOT / a.output).parent.mkdir(parents=True, exist_ok=True) (ROOT / a.output).write_text(text + '\n', encoding='utf-8') if res['result'] != 'PASS': raise SystemExit(1) if __name__ == '__main__': main()