forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
168 lines
9.6 KiB
Python
168 lines
9.6 KiB
Python
"""Compare the entire proxy/direct upload workflows, including complete validation."""
|
|
import argparse
|
|
import importlib.util
|
|
import datetime
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify
|
|
|
|
spec = importlib.util.spec_from_file_location('upload_metrics', ROOT / 'test/concurrent-uploads.py')
|
|
metrics = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(metrics)
|
|
|
|
|
|
def collect_profiles(case, since, repeats):
|
|
records = {}
|
|
for line in docker('compose', 'logs', '--no-color', '--since', since, 'api').splitlines():
|
|
if 'UPLOAD_MEMORY_PROFILE ' in line:
|
|
record = json.loads(line.split('UPLOAD_MEMORY_PROFILE ', 1)[1])
|
|
records[record['requestId']] = record
|
|
case['profiles'] = [records[request_id] for request_id in case['requestIds'] if request_id in records]
|
|
check(len(case['profiles']) == repeats, 'Missing opt-in API stage profiles')
|
|
for record in case['profiles']:
|
|
check(record['mode'] == case['mode'] and record['status'] == 201 and record['finished'], 'Invalid stage profile')
|
|
names = {stage['name'] for stage in record['stages']}
|
|
check({'sharp_metadata', 'sharp_decode_stats', 'serialize_response'} <= names, 'Async stage context was lost')
|
|
if case['mode'] == 'direct':
|
|
check({'s3_stream_sha256_to_file', 'sha256_finalize', 's3_copy'} <= names, 'Missing direct profile stages')
|
|
|
|
|
|
def restart_api(client, containers):
|
|
def states():
|
|
return docker('inspect', '--format', '{{.State.Health.Status}} {{.State.StartedAt}}', *containers).splitlines()
|
|
before = states()
|
|
docker('compose', 'restart', 'api')
|
|
deadline = time.monotonic() + 120
|
|
routed = set()
|
|
while time.monotonic() < deadline:
|
|
current = states()
|
|
if len(current) == 2 and all(line.startswith('healthy ') for line in current):
|
|
check(all(old != new for old, new in zip(before, current)), 'API did not restart')
|
|
try:
|
|
response = client.call('GET', '/health/ready')
|
|
routed.add(response['headers']['X-Instance-Id'])
|
|
if len(routed) == 2:
|
|
return current
|
|
except Failure:
|
|
pass
|
|
time.sleep(1)
|
|
raise Failure('API did not become ready after restart')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792])
|
|
parser.add_argument('--repeats', type=int, default=3)
|
|
parser.add_argument('--restart-api', action='store_true', help='Restart both API replicas before each measured case')
|
|
parser.add_argument('--rounds', type=int, default=1, help='Reverse workflow order on alternate rounds')
|
|
parser.add_argument('--profile', action='store_true', help='Collect opt-in API stage profiles and RSS after a five-second idle period')
|
|
args = parser.parse_args()
|
|
check(len(set(args.sizes)) >= 2 and all(64 <= side <= 1800 for side in args.sizes), 'Choose at least two image sides between 64 and 1800')
|
|
check(1 <= args.repeats <= 10, 'Repeats must be between 1 and 10')
|
|
check(1 <= args.rounds <= 5, 'Rounds must be between 1 and 5')
|
|
warmups = 4 if args.restart_api else 1
|
|
report = {'status': 'failed', 'repeats': args.repeats, 'rounds': args.rounds,
|
|
'restartApiBeforeCase': args.restart_api, 'warmupUploadsPerCase': warmups, 'cases': [],
|
|
'method': 'One upload at a time; measured direct flow includes grant, PUT, and complete with full image validation. Download verification is outside the measurement window. RSS sampling 50 ms; network counters sum non-loopback API interfaces, including DB/S3/background health traffic. Probe RSS excluded.'}
|
|
probes = []
|
|
log_start = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
try:
|
|
readiness = Client('http://127.0.0.1:8080', timeout=5)
|
|
deadline = time.monotonic() + 90
|
|
routed = set()
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id'])
|
|
if len(routed) == 2: break
|
|
except Failure:
|
|
pass
|
|
time.sleep(1)
|
|
check(len(routed) == 2, 'Both API replicas must be ready through Nginx')
|
|
with fixture() as (client, owner, reader, chat, _):
|
|
containers = docker('compose', 'ps', '-q', 'api').split()
|
|
check(len(containers) == 2, 'Expected two API replicas')
|
|
cases = [(round_index + 1, side, mode) for round_index in range(args.rounds)
|
|
for side in args.sizes
|
|
for mode in (['proxy', 'direct'] if round_index % 2 == 0 else ['direct', 'proxy'])]
|
|
for round_number, side, mode in cases:
|
|
image = metrics.png_image(side) + metrics.chunk(b'IEND', b'')
|
|
for probe in probes: probe.close()
|
|
probes.clear()
|
|
restarted = restart_api(client, containers) if args.restart_api else None
|
|
for container in containers:
|
|
probe = metrics.MemoryProbe(container)
|
|
probes.append(probe)
|
|
probe.ready = probe.receive('ready')
|
|
def upload():
|
|
if mode == 'direct':
|
|
cap = grant(client, owner, chat, image)
|
|
put(client, cap, image)
|
|
response = complete(client, owner, cap)
|
|
else:
|
|
response = client.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=benchmark.png", 201,
|
|
actor=owner, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
return response['json'], response['headers']['X-Instance-Id'], response['headers']['X-Request-Id']
|
|
# Warm the selected workflow before recording it. Warm-up failures fail the run.
|
|
warmed = set()
|
|
for warmup_index in range(warmups):
|
|
# Vary request parity so least_conn does not pin every complete
|
|
# to the same replica in the four-request direct workflow.
|
|
if args.restart_api and warmup_index % 2:
|
|
client.call('GET', '/health/live')
|
|
attachment, instance, _ = upload()
|
|
warmed.add(instance)
|
|
verify(client, reader, attachment, image)
|
|
if args.restart_api:
|
|
check(warmed == {probe.ready['instance'] for probe in probes}, 'Warm-up did not cover both API replicas')
|
|
for probe in probes: probe.command('begin', 'begin')
|
|
started = time.monotonic()
|
|
attachments = []
|
|
request_ids = []
|
|
try:
|
|
for _ in range(args.repeats):
|
|
attachment, _, request_id = upload()
|
|
attachments.append(attachment)
|
|
request_ids.append(request_id)
|
|
finally:
|
|
elapsed = time.monotonic() - started
|
|
samples = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes]
|
|
report['cases'].append({'round': round_number, 'mode': mode, 'sidePixels': side, 'fileBytes': len(image),
|
|
'apiStatesAfterRestart': restarted, 'warmupInstances': sorted(warmed),
|
|
'requestIds': request_ids,
|
|
'successfulUploads': len(attachments), 'elapsedSeconds': elapsed,
|
|
'memory': samples, 'apiRxBytes': sum(m['networkRxBytes'] for m in samples),
|
|
'apiTxBytes': sum(m['networkTxBytes'] for m in samples)})
|
|
for sample in samples:
|
|
check(sample['samples'] >= 2 and sample['oomDelta'] == sample['oomKillDelta'] == 0, 'Invalid memory samples or OOM')
|
|
if args.profile:
|
|
time.sleep(5)
|
|
report['cases'][-1]['idleAfterFiveSeconds'] = [
|
|
{'instance': probe.ready['instance'], **probe.command('snapshot', 'snapshot')} for probe in probes]
|
|
collect_profiles(report['cases'][-1], log_start, args.repeats)
|
|
for attachment in attachments: verify(client, reader, attachment, image)
|
|
print(f"PASS: round {round_number}, {mode}, {side}px, {len(image)} bytes, {len(attachments)} uploads; API RX/TX={sum(m['networkRxBytes'] for m in samples)}/{sum(m['networkTxBytes'] for m in samples)}", flush=True)
|
|
for probe in probes: probe.close()
|
|
probes.clear()
|
|
report['fixturesRemoved'] = True
|
|
report['status'] = 'passed'
|
|
except BaseException as error:
|
|
report['errorType'] = type(error).__name__
|
|
if isinstance(error, Failure): report['failure'] = str(error)
|
|
raise
|
|
finally:
|
|
for probe in probes: probe.close()
|
|
filename = 'direct-upload-memory-profile.json' if args.profile else (
|
|
'direct-upload-benchmark-restarted.json' if args.restart_api else 'direct-upload-benchmark.json')
|
|
(ROOT / 'evidence' / filename).write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (Failure, ValueError, OSError, subprocess.SubprocessError) as error:
|
|
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
|
|
sys.exit(1)
|