"""Executed via Docker exec stdin. Prints only sanitized measurements as JSON.""" import argparse import base64 from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import hashlib import io import json import math import os from pathlib import Path import random import sys import time import urllib.error import urllib.request from uuid import uuid4 sys.path.insert(0, '/app') from PIL import Image from sqlalchemy import select from sqlalchemy.orm import sessionmaker from src.database import create_db_engine from src.models import Attachment, AttachmentJob from src.object_storage import BUCKET, s3_client BASE = 'http://nginx:8080' def call(method, path, body=None, token=None, raw=None, headers=None): headers = dict(headers or {}) if token: headers['Authorization'] = 'Bearer ' + token if body is not None: raw = json.dumps(body).encode() headers['Content-Type'] = 'application/json' request = urllib.request.Request(BASE + path, data=raw, headers=headers, method=method) try: response = urllib.request.urlopen(request, timeout=60) except urllib.error.HTTPError as error: response = error with response: data = response.read() payload = json.loads(data) if response.headers.get_content_type() == 'application/json' else data return response.status, payload, response.headers def bootstrap(): username, password = 'q_' + uuid4().hex[:20], uuid4().hex status, user, _ = call('POST', '/api/v1/users', body={ 'username': username, 'password': password, 'display_name': 'Synthetic queue experiment'}) assert status == 201, ('user', status) basic = base64.b64encode(f'{username}:{password}'.encode()).decode() status, session, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic}) assert status == 201, ('session', status) token = session['token'] status, chat, _ = call('POST', '/api/v1/chats', token=token, body={'title': 'Synthetic queue experiment', 'member_ids': [user['id']]}) assert status == 201, ('chat', status) return user['id'], chat['id'], token def fixture(width, height): pixels = random.Random(202605).randbytes(width * height * 3) with Image.frombytes('RGB', (width, height), pixels) as picture, io.BytesIO() as output: picture.save(output, format='JPEG', quality=85) return output.getvalue() def percentile(values): return sorted(values)[max(0, math.ceil(len(values) * 0.95) - 1)] def rss_mib(): # Sum process RSS, not cgroup memory.current (which also counts page cache). # Shared pages may be counted twice. Exclude this measurement process. total = 0 for path in Path('/proc').glob('[0-9]*/status'): if path.parent.name == str(os.getpid()): continue try: for line in path.read_text().splitlines(): if line.startswith('VmRSS:'): total += int(line.split()[1]) except (FileNotFoundError, ProcessLookupError): pass return total / 1024 def rows_for(sessions, owner_id): with sessions() as session: return list(session.execute(select(Attachment, AttachmentJob).join( AttachmentJob, AttachmentJob.attachment_id == Attachment.id) .where(Attachment.owner_id == owner_id))) def summary(rows): states = {key: 0 for key in ('queued', 'processing', 'ready', 'failed')} ages = [] for attachment, _ in rows: states[attachment.status] += 1 if attachment.status == 'queued': ages.append((datetime.now(timezone.utc) - attachment.created_at).total_seconds()) return {**states, 'oldest_queued_s': max(ages, default=0)} def staging_keys(): return {obj['Key'] for page in s3_client().get_paginator('list_objects_v2').paginate( Bucket=BUCKET, Prefix='staging/') for obj in page.get('Contents', [])} def upload_batch(args, sessions): owner, chat, token = bootstrap() data = fixture(args.width, args.height) before_staging = staging_keys() if args.mode == 'overload' else set() samples = [] started = time.monotonic() def upload(_): return call('POST', f'/api/v1/chats/{chat}/attachments?filename=load.jpg', token=token, raw=data, headers={'Content-Type': 'image/jpeg'}) with ThreadPoolExecutor(max_workers=args.clients) as pool: futures = [pool.submit(upload, i) for i in range(args.count)] while True: rows = rows_for(sessions, owner) snapshot = summary(rows) samples.append({'elapsed_s': round(time.monotonic() - started, 3), 'rss_mib': round(rss_mib(), 3), **snapshot}) if all(f.done() for f in futures): responses = [f.result() for f in futures] accepted = [payload for status, payload, _ in responses if status == 202] if args.mode == 'overload': rows = rows_for(sessions, owner) snapshot = summary(rows) if args.mode == 'overload' or snapshot['ready'] + snapshot['failed'] == len(accepted): break if time.monotonic() - started > 300: raise AssertionError('Batch did not drain in 300 seconds') time.sleep(0.1) elapsed = time.monotonic() - started codes = {str(code): sum(status == code for status, _, _ in responses) for code in sorted({status for status, _, _ in responses})} replicas = len({headers['X-Instance-Id'] for _, _, headers in responses}) result = {'width': args.width, 'height': args.height, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'requests': args.count, 'clients': args.clients, 'http_statuses': codes, 'api_replicas_seen': replicas, 'elapsed_s': elapsed, 'owner_id': owner, 'attachment_ids': [a['id'] for a in accepted]} assert len(rows) == len(accepted), 'Rejected upload created metadata' assert replicas == 2, 'Experiment must exercise both API replicas' if args.mode == 'overload': assert len(accepted) == args.limit, (codes, args.limit) assert codes.get('429') == args.count - args.limit, codes assert all(a.status == 'queued' for a, _ in rows), 'Worker was not stopped' assert staging_keys() - before_staging == {'staging/' + a['id'] for a in accepted}, 'Rejected upload left S3 object' for status, payload, headers in responses: if status == 429: assert payload['error']['code'] == 'rate_limited' assert int(headers['Retry-After']) > 0 assert call('GET', '/health/ready')[0] == 200 assert call('GET', f"/api/v1/attachments/{accepted[0]['id']}/content", token=token)[0] == 409 assert call('POST', f'/api/v1/chats/{chat}/messages', token=token, body={'text': 'Still responsive under overload'}, headers={'Idempotency-Key': uuid4().hex})[0] == 201 result.update({'limit': args.limit, 'accepted': len(accepted), 'rejected': codes['429'], 'health_and_messages_available': True, 'rejected_leave_no_objects': True}) else: assert codes == {'202': args.count}, codes assert all(a.status == 'ready' and j.attempt == 1 for a, j in rows), 'Failed/retried job in baseline' waits = [(j.first_started_at - j.created_at).total_seconds() for _, j in rows] latencies = [(a.updated_at - a.created_at).total_seconds() for a, _ in rows] result.update({'throughput_per_s': len(rows) / elapsed, 'ready_p95_s': percentile(latencies), 'queue_wait_p95_s': percentile(waits), 'queue_lag_max_jobs': max(s['queued'] for s in samples), 'queue_oldest_max_s': max(s['oldest_queued_s'] for s in samples), 'processing_max': max(s['processing'] for s in samples), 'rss_peak_mib': max(s['rss_mib'] for s in samples), 'samples': samples}) return result def recover(args, sessions): expected = json.loads(args.expected) started = time.monotonic() while time.monotonic() - started < 180: rows = rows_for(sessions, expected['owner_id']) assert not any(a.status == 'failed' for a, _ in rows), 'Accepted job failed' if len(rows) == expected['accepted'] and all(a.status == 'ready' for a, _ in rows): break time.sleep(0.2) else: raise AssertionError('Accepted work did not recover') assert {a.id for a, _ in rows} == set(expected['attachment_ids']), 'Lost/replaced accepted IDs' for attachment, _ in rows: keys = [attachment.object_key] + [v['object_key'] for v in attachment.variants] assert len(attachment.variants) == 3 for key in keys: s3_client().head_object(Bucket=BUCKET, Key=key) with s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] as body: assert hashlib.sha256(body.read()).hexdigest() == expected['sha256'] elapsed = time.monotonic() - started # A new accepted upload proves capacity is released after draining. # One request need not hit both replicas, so perform this explicitly. owner, chat, token = bootstrap() status, _, _ = call('POST', f'/api/v1/chats/{chat}/attachments?filename=after.jpg', token=token, raw=fixture(320, 200), headers={'Content-Type': 'image/jpeg'}) assert status == 202, 'Admission did not recover' deadline = time.monotonic() + 60 while time.monotonic() < deadline: fresh = rows_for(sessions, owner) if fresh and fresh[0][0].status == 'ready': break time.sleep(0.2) else: raise AssertionError('New upload did not complete after overload') return {'accepted_ids_preserved': True, 'all_ready': len(rows), 'original_checksums_match': True, 'all_variants_exist': True, 'drain_observed_s': elapsed, 'new_upload_status': status} def main(): parser = argparse.ArgumentParser() parser.add_argument('mode', choices=['measure', 'overload', 'recover']) parser.add_argument('--width', type=int, default=320) parser.add_argument('--height', type=int, default=200) parser.add_argument('--count', type=int, default=18) parser.add_argument('--clients', type=int, default=6) parser.add_argument('--limit', type=int, default=12) parser.add_argument('--expected') args = parser.parse_args() engine = create_db_engine() try: sessions = sessionmaker(engine, expire_on_commit=False) result = recover(args, sessions) if args.mode == 'recover' else upload_batch(args, sessions) print(json.dumps(result)) finally: engine.dispose() if __name__ == '__main__': main()