"""Compare the entire proxy/direct upload workflows, including complete validation.""" import argparse import importlib.util import datetime import json import subprocess import sys import time from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify spec = importlib.util.spec_from_file_location('upload_metrics', ROOT / 'test/concurrent-uploads.py') metrics = importlib.util.module_from_spec(spec) spec.loader.exec_module(metrics) def collect_profiles(case, since, repeats): records = {} for line in docker('compose', 'logs', '--no-color', '--since', since, 'api').splitlines(): if 'UPLOAD_MEMORY_PROFILE ' in line: record = json.loads(line.split('UPLOAD_MEMORY_PROFILE ', 1)[1]) records[record['requestId']] = record case['profiles'] = [records[request_id] for request_id in case['requestIds'] if request_id in records] check(len(case['profiles']) == repeats, 'Missing opt-in API stage profiles') for record in case['profiles']: check(record['mode'] == case['mode'] and record['status'] == 201 and record['finished'], 'Invalid stage profile') names = {stage['name'] for stage in record['stages']} check({'sharp_metadata', 'sharp_decode_stats', 'serialize_response'} <= names, 'Async stage context was lost') if case['mode'] == 'direct': check({'s3_stream_sha256_to_file', 'sha256_finalize', 's3_copy'} <= names, 'Missing direct profile stages') def restart_api(client, containers): def states(): return docker('inspect', '--format', '{{.State.Health.Status}} {{.State.StartedAt}}', *containers).splitlines() before = states() docker('compose', 'restart', 'api') deadline = time.monotonic() + 120 routed = set() while time.monotonic() < deadline: current = states() if len(current) == 2 and all(line.startswith('healthy ') for line in current): check(all(old != new for old, new in zip(before, current)), 'API did not restart') try: response = client.call('GET', '/health/ready') routed.add(response['headers']['X-Instance-Id']) if len(routed) == 2: return current except Failure: pass time.sleep(1) raise Failure('API did not become ready after restart') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792]) parser.add_argument('--repeats', type=int, default=3) parser.add_argument('--restart-api', action='store_true', help='Restart both API replicas before each measured case') parser.add_argument('--rounds', type=int, default=1, help='Reverse workflow order on alternate rounds') parser.add_argument('--profile', action='store_true', help='Collect opt-in API stage profiles and RSS after a five-second idle period') args = parser.parse_args() check(len(set(args.sizes)) >= 2 and all(64 <= side <= 1800 for side in args.sizes), 'Choose at least two image sides between 64 and 1800') check(1 <= args.repeats <= 10, 'Repeats must be between 1 and 10') check(1 <= args.rounds <= 5, 'Rounds must be between 1 and 5') warmups = 4 if args.restart_api else 1 report = {'status': 'failed', 'repeats': args.repeats, 'rounds': args.rounds, 'restartApiBeforeCase': args.restart_api, 'warmupUploadsPerCase': warmups, 'cases': [], 'method': 'One upload at a time; measured direct flow includes grant, PUT, and complete with full image validation. Download verification is outside the measurement window. RSS sampling 50 ms; network counters sum non-loopback API interfaces, including DB/S3/background health traffic. Probe RSS excluded.'} probes = [] log_start = datetime.datetime.now(datetime.timezone.utc).isoformat() try: readiness = Client('http://127.0.0.1:8080', timeout=5) deadline = time.monotonic() + 90 routed = set() while time.monotonic() < deadline: try: routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id']) if len(routed) == 2: break except Failure: pass time.sleep(1) check(len(routed) == 2, 'Both API replicas must be ready through Nginx') with fixture() as (client, owner, reader, chat, _): containers = docker('compose', 'ps', '-q', 'api').split() check(len(containers) == 2, 'Expected two API replicas') cases = [(round_index + 1, side, mode) for round_index in range(args.rounds) for side in args.sizes for mode in (['proxy', 'direct'] if round_index % 2 == 0 else ['direct', 'proxy'])] for round_number, side, mode in cases: image = metrics.png_image(side) + metrics.chunk(b'IEND', b'') for probe in probes: probe.close() probes.clear() restarted = restart_api(client, containers) if args.restart_api else None for container in containers: probe = metrics.MemoryProbe(container) probes.append(probe) probe.ready = probe.receive('ready') def upload(): if mode == 'direct': cap = grant(client, owner, chat, image) put(client, cap, image) response = complete(client, owner, cap) else: response = client.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=benchmark.png", 201, actor=owner, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment') return response['json'], response['headers']['X-Instance-Id'], response['headers']['X-Request-Id'] # Warm the selected workflow before recording it. Warm-up failures fail the run. warmed = set() for warmup_index in range(warmups): # Vary request parity so least_conn does not pin every complete # to the same replica in the four-request direct workflow. if args.restart_api and warmup_index % 2: client.call('GET', '/health/live') attachment, instance, _ = upload() warmed.add(instance) verify(client, reader, attachment, image) if args.restart_api: check(warmed == {probe.ready['instance'] for probe in probes}, 'Warm-up did not cover both API replicas') for probe in probes: probe.command('begin', 'begin') started = time.monotonic() attachments = [] request_ids = [] try: for _ in range(args.repeats): attachment, _, request_id = upload() attachments.append(attachment) request_ids.append(request_id) finally: elapsed = time.monotonic() - started samples = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes] report['cases'].append({'round': round_number, 'mode': mode, 'sidePixels': side, 'fileBytes': len(image), 'apiStatesAfterRestart': restarted, 'warmupInstances': sorted(warmed), 'requestIds': request_ids, 'successfulUploads': len(attachments), 'elapsedSeconds': elapsed, 'memory': samples, 'apiRxBytes': sum(m['networkRxBytes'] for m in samples), 'apiTxBytes': sum(m['networkTxBytes'] for m in samples)}) for sample in samples: check(sample['samples'] >= 2 and sample['oomDelta'] == sample['oomKillDelta'] == 0, 'Invalid memory samples or OOM') if args.profile: time.sleep(5) report['cases'][-1]['idleAfterFiveSeconds'] = [ {'instance': probe.ready['instance'], **probe.command('snapshot', 'snapshot')} for probe in probes] collect_profiles(report['cases'][-1], log_start, args.repeats) for attachment in attachments: verify(client, reader, attachment, image) print(f"PASS: round {round_number}, {mode}, {side}px, {len(image)} bytes, {len(attachments)} uploads; API RX/TX={sum(m['networkRxBytes'] for m in samples)}/{sum(m['networkTxBytes'] for m in samples)}", flush=True) for probe in probes: probe.close() probes.clear() report['fixturesRemoved'] = True report['status'] = 'passed' except BaseException as error: report['errorType'] = type(error).__name__ if isinstance(error, Failure): report['failure'] = str(error) raise finally: for probe in probes: probe.close() filename = 'direct-upload-memory-profile.json' if args.profile else ( 'direct-upload-benchmark-restarted.json' if args.restart_api else 'direct-upload-benchmark.json') (ROOT / 'evidence' / filename).write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') if __name__ == '__main__': try: main() except (Failure, ValueError, OSError, subprocess.SubprocessError) as error: print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr) sys.exit(1)