"""Compare proxy/direct uploads under synchronized load; measure the main API processes.""" import argparse import base64 from concurrent.futures import ThreadPoolExecutor import hashlib import http.client import importlib.util import json import statistics import subprocess import sys import threading import time from urllib.parse import urlsplit from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify spec = importlib.util.spec_from_file_location('direct_benchmark', ROOT / 'test/direct-upload-benchmark.py') benchmark = importlib.util.module_from_spec(spec) spec.loader.exec_module(benchmark) metrics = benchmark.metrics class HttpFailure(Failure): def __init__(self, stage, status, headers): super().__init__(f'{stage} returned HTTP {status}') self.details = {'stage': stage, 'status': status, 'instance': headers.get('X-Instance-Id'), 'requestId': headers.get('X-Request-Id')} def paced_request(method, url, headers, image, gate, duration): target = urlsplit(url) check(target.scheme == 'http', 'This benchmark targets the local HTTP stack') connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60) try: connection.putrequest(method, target.path + ('?' + target.query if target.query else '')) for name, value in headers.items(): connection.putheader(name, value) connection.endheaders() first = min(65536, len(image) - 1) connection.send(image[:first]) gate.wait(timeout=60) started = time.monotonic() for offset in range(first, len(image), 65536): end = min(offset + 65536, len(image)) deadline = started + duration * (end - first) / (len(image) - first) time.sleep(max(0, deadline - time.monotonic())) connection.send(image[offset:end]) response = connection.getresponse() body = response.read(1024 * 1024 + 1) check(len(body) <= 1024 * 1024, 'Unexpectedly large response') return response.status, response.headers, body finally: connection.close() def error_result(error, stage): # Never serialize exceptions containing a signed URL or request headers. return {'ok': False, 'stage': stage, 'errorType': type(error).__name__, **getattr(error, 'details', {})} def upload(client, owner, chat, image, digest, mode, gate, completion_gate, duration): started = time.monotonic() error = None attachment = None instance = None stage = 'grant' if mode == 'direct' else 'proxy' try: if mode == 'direct': capability = grant(client, owner, chat, image) stage = 'put' status, headers, _ = paced_request('PUT', capability['url'], capability['headers'], image, gate, duration) if status != 200: raise HttpFailure(stage, status, headers) else: status, headers, body = paced_request( 'POST', client.base + f"/api/v1/chats/{chat['id']}/attachments?filename=load.png", {'Authorization': 'Bearer ' + owner['token'], 'Content-Type': 'image/png', 'Content-Length': str(len(image))}, image, gate, duration) if status != 201: raise HttpFailure(stage, status, headers) attachment = metrics.shape(json.loads(body), 'Attachment') instance = headers.get('X-Instance-Id') except Exception as caught: error = error_result(caught, stage) gate.abort() if mode == 'direct': # All workers arrive even if their PUT failed; successful PUTs can still complete. try: completion_gate.wait(timeout=65) if error is None: stage = 'complete' response = complete(client, owner, capability, [201, 400, 401, 404, 408, 409, 413, 415, 429, 500, 503]) if response['status'] != 201: raise HttpFailure(stage, response['status'], response['headers']) attachment = metrics.shape(json.loads(response['bytes']), 'Attachment') instance = response['headers'].get('X-Instance-Id') except Exception as caught: if error is None: error = error_result(caught, stage) if error is None: try: check(attachment['sha256'] == digest and attachment['size_bytes'] == len(image), 'Incorrect checksum/size') check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == owner['id'], 'Incorrect ownership') check(bool(instance), 'Missing instance ID') except Exception as caught: error = error_result(caught, 'validate_response') return {**(error or {'ok': True, 'attachment': attachment, 'instance': instance}), 'elapsedSeconds': time.monotonic() - started} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--sizes', type=int, nargs='+', default=[1792]) parser.add_argument('--concurrency', type=int, nargs='+', default=[1, 5, 10]) parser.add_argument('--rounds', type=int, default=2) parser.add_argument('--waves', type=int, default=2) parser.add_argument('--upload-seconds', type=float, default=2) args = parser.parse_args() check(all(64 <= n <= 1800 for n in args.sizes), 'Invalid image size') check(all(1 <= n <= 20 for n in args.concurrency), 'Concurrency must be 1..20') check(1 <= args.rounds <= 3 and 1 <= args.waves <= 5, 'Invalid rounds/waves') check(0.1 <= args.upload_seconds <= 5, 'Pacing must be 0.1..5 seconds') report = {'status': 'running', 'cases': [], 'settings': vars(args), 'method': 'Fresh API restart per case, exactly 2 verified warm-up uploads per replica via internal API addresses; temporary docker-exec client exits before measurement. Measured uploads use public Nginx. Two paced waves by default; all bodies are incomplete at the upload barrier, direct completes start at a second barrier. Full workflow latency includes grant/PUT/complete. External per-API-process RSS sampling at 50 ms; cgroup memory recorded separately. Independent replica peaks are not summed. No retries. Download verification is outside the measurement window.'} output = ROOT / 'evidence/direct-upload-load.json' def save(): output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') probes = [] try: readiness = Client('http://127.0.0.1:8080', timeout=5) deadline = time.monotonic() + 90 routed = set() while time.monotonic() < deadline: try: routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id']) except Failure: pass if len(routed) == 2: break time.sleep(1) check(len(routed) == 2, 'Both API replicas must be ready') with fixture() as (client, owner, reader, chat, _): containers = docker('compose', 'ps', '-q', 'api').split() check(len(containers) == 2, 'Expected two replicas') for container in containers: flag = docker('exec', container, 'node', '-p', "process.env.UPLOAD_MEMORY_PROFILE || '0'").strip() check(flag == '0', 'Disable detailed profiling for the load comparison') cases = [(r + 1, side, concurrency, mode) for r in range(args.rounds) for side in args.sizes for concurrency in args.concurrency for mode in (['proxy', 'direct'] if r % 2 == 0 else ['direct', 'proxy'])] for round_number, side, concurrency, mode in cases: image = metrics.png_image(side) + metrics.chunk(b'IEND', b'') digest = hashlib.sha256(image).hexdigest() for probe in probes: probe.close() probes.clear() restarted = benchmark.restart_api(client, containers) for container in containers: probe = metrics.MemoryProbe(container) probes.append(probe) probe.ready = probe.receive('ready') target_rows = docker('inspect', '--format', '{{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}', *containers).splitlines() targets = [{'instance': row.split()[0], 'address': row.split()[1]} for row in target_rows] source = (ROOT / 'test/warm-upload-replicas.mjs').read_text(encoding='utf-8') warmup = json.loads(docker('exec', '-i', containers[0], 'node', '--input-type=module', '-e', source, input=json.dumps({'targets': targets, 'owner': owner, 'reader': reader, 'chat': chat, 'imageBase64': base64.b64encode(image).decode(), 'mode': mode}))) check(warmup['counts'] == {p.ready['instance']: 2 for p in probes}, 'Unequal per-replica warm-up') identities = docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers) workers = [Client(client.base, timeout=60) for _ in range(concurrency)] case = {'round': round_number, 'sidePixels': side, 'fileBytes': len(image), 'mode': mode, 'concurrency': concurrency, 'apiStatesAfterRestart': restarted, 'warmupCounts': warmup['counts'], 'waves': [], 'verifiedDownloads': 0} report['cases'].append(case) results = [] for probe in probes: probe.command('begin', 'begin') started = time.monotonic() try: with ThreadPoolExecutor(max_workers=concurrency) as pool: for wave in range(args.waves): gate = threading.Barrier(concurrency + 1) complete_gate = threading.Barrier(concurrency) futures = [pool.submit(upload, worker, owner, chat, image, digest, mode, gate, complete_gate, args.upload_seconds) for worker in workers] overlapped = False try: gate.wait(timeout=60) overlapped = True except threading.BrokenBarrierError: pass wave_results = [future.result() for future in futures] results.extend(wave_results) case['waves'].append({'wave': wave + 1, 'incompleteBodiesAtBarrier': concurrency if overlapped else 0, 'attempts': [{k: v for k, v in result.items() if k != 'attachment'} for result in wave_results]}) finally: case['elapsedSeconds'] = time.monotonic() - started case['memory'] = [{'instance': p.ready['instance'], **p.command('end', 'summary')} for p in probes] case['successfulUploads'] = sum(result['ok'] for result in results) case['failedUploads'] = len(results) - case['successfulUploads'] case['successfulUploadsPerSecond'] = case['successfulUploads'] / case['elapsedSeconds'] case['instances'] = {name: sum(r['ok'] and r.get('instance') == name for r in results) for name in sorted({r['instance'] for r in results if r['ok']})} latencies = sorted(result['elapsedSeconds'] for result in results) case['latencySeconds'] = {'median': statistics.median(latencies), 'max': max(latencies)} case['apiIdentityUnchanged'] = identities == docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers) save() check(case['apiIdentityUnchanged'], 'API restarted during load') for sample in case['memory']: check(sample['samples'] >= 2, 'Insufficient RSS samples') check(sample['oomDelta'] == sample['oomKillDelta'] == 0, 'API OOM; stopping further load') for result in results: if result['ok']: verify(client, reader, result['attachment'], image) case['verifiedDownloads'] += 1 check(len({r['attachment']['id'] for r in results if r['ok']}) == case['successfulUploads'], 'Duplicate attachment IDs') if concurrency >= 5 and case['failedUploads'] == 0: check(len(case['instances']) == 2, 'Concurrent work did not reach both replicas') try: case['apiReadyAfterCase'] = client.call('GET', '/health/ready')['status'] == 200 except Failure: case['apiReadyAfterCase'] = False save() peaks = ', '.join(f"{m['sampledPeakRssBytes'] / 2**20:.1f}" for m in case['memory']) print(f"DONE: round {round_number}, {mode}, {side}px, N={concurrency}, {case['successfulUploads']}/{len(results)}, RSS MiB=[{peaks}]", flush=True) for probe in probes: probe.close() probes.clear() report['fixturesRemoved'] = True report['status'] = 'completed_with_errors' if any(c['failedUploads'] or not c['apiReadyAfterCase'] for c in report['cases']) else 'passed' return report['status'] == 'passed' except BaseException as error: report['status'] = 'failed' report['errorType'] = type(error).__name__ if isinstance(error, Failure): report['failure'] = str(error) raise finally: for probe in probes: probe.close() save() if __name__ == '__main__': try: sys.exit(0 if main() else 1) except (Failure, ValueError, OSError, subprocess.SubprocessError) as error: print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr) sys.exit(1)