forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
238 lines
14 KiB
Python
238 lines
14 KiB
Python
"""Compare proxy/direct uploads under synchronized load; measure the main API processes."""
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import http.client
|
|
import importlib.util
|
|
import json
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
|
|
from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify
|
|
|
|
spec = importlib.util.spec_from_file_location('direct_benchmark', ROOT / 'test/direct-upload-benchmark.py')
|
|
benchmark = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(benchmark)
|
|
metrics = benchmark.metrics
|
|
|
|
|
|
class HttpFailure(Failure):
|
|
def __init__(self, stage, status, headers):
|
|
super().__init__(f'{stage} returned HTTP {status}')
|
|
self.details = {'stage': stage, 'status': status,
|
|
'instance': headers.get('X-Instance-Id'), 'requestId': headers.get('X-Request-Id')}
|
|
|
|
|
|
def paced_request(method, url, headers, image, gate, duration):
|
|
target = urlsplit(url)
|
|
check(target.scheme == 'http', 'This benchmark targets the local HTTP stack')
|
|
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60)
|
|
try:
|
|
connection.putrequest(method, target.path + ('?' + target.query if target.query else ''))
|
|
for name, value in headers.items(): connection.putheader(name, value)
|
|
connection.endheaders()
|
|
first = min(65536, len(image) - 1)
|
|
connection.send(image[:first])
|
|
gate.wait(timeout=60)
|
|
started = time.monotonic()
|
|
for offset in range(first, len(image), 65536):
|
|
end = min(offset + 65536, len(image))
|
|
deadline = started + duration * (end - first) / (len(image) - first)
|
|
time.sleep(max(0, deadline - time.monotonic()))
|
|
connection.send(image[offset:end])
|
|
response = connection.getresponse()
|
|
body = response.read(1024 * 1024 + 1)
|
|
check(len(body) <= 1024 * 1024, 'Unexpectedly large response')
|
|
return response.status, response.headers, body
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def error_result(error, stage):
|
|
# Never serialize exceptions containing a signed URL or request headers.
|
|
return {'ok': False, 'stage': stage, 'errorType': type(error).__name__,
|
|
**getattr(error, 'details', {})}
|
|
|
|
|
|
def upload(client, owner, chat, image, digest, mode, gate, completion_gate, duration):
|
|
started = time.monotonic()
|
|
error = None
|
|
attachment = None
|
|
instance = None
|
|
stage = 'grant' if mode == 'direct' else 'proxy'
|
|
try:
|
|
if mode == 'direct':
|
|
capability = grant(client, owner, chat, image)
|
|
stage = 'put'
|
|
status, headers, _ = paced_request('PUT', capability['url'], capability['headers'], image, gate, duration)
|
|
if status != 200: raise HttpFailure(stage, status, headers)
|
|
else:
|
|
status, headers, body = paced_request(
|
|
'POST', client.base + f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
|
|
{'Authorization': 'Bearer ' + owner['token'], 'Content-Type': 'image/png',
|
|
'Content-Length': str(len(image))}, image, gate, duration)
|
|
if status != 201: raise HttpFailure(stage, status, headers)
|
|
attachment = metrics.shape(json.loads(body), 'Attachment')
|
|
instance = headers.get('X-Instance-Id')
|
|
except Exception as caught:
|
|
error = error_result(caught, stage)
|
|
gate.abort()
|
|
if mode == 'direct':
|
|
# All workers arrive even if their PUT failed; successful PUTs can still complete.
|
|
try:
|
|
completion_gate.wait(timeout=65)
|
|
if error is None:
|
|
stage = 'complete'
|
|
response = complete(client, owner, capability, [201, 400, 401, 404, 408, 409, 413, 415, 429, 500, 503])
|
|
if response['status'] != 201: raise HttpFailure(stage, response['status'], response['headers'])
|
|
attachment = metrics.shape(json.loads(response['bytes']), 'Attachment')
|
|
instance = response['headers'].get('X-Instance-Id')
|
|
except Exception as caught:
|
|
if error is None: error = error_result(caught, stage)
|
|
if error is None:
|
|
try:
|
|
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(image), 'Incorrect checksum/size')
|
|
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == owner['id'], 'Incorrect ownership')
|
|
check(bool(instance), 'Missing instance ID')
|
|
except Exception as caught:
|
|
error = error_result(caught, 'validate_response')
|
|
return {**(error or {'ok': True, 'attachment': attachment, 'instance': instance}),
|
|
'elapsedSeconds': time.monotonic() - started}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--sizes', type=int, nargs='+', default=[1792])
|
|
parser.add_argument('--concurrency', type=int, nargs='+', default=[1, 5, 10])
|
|
parser.add_argument('--rounds', type=int, default=2)
|
|
parser.add_argument('--waves', type=int, default=2)
|
|
parser.add_argument('--upload-seconds', type=float, default=2)
|
|
args = parser.parse_args()
|
|
check(all(64 <= n <= 1800 for n in args.sizes), 'Invalid image size')
|
|
check(all(1 <= n <= 20 for n in args.concurrency), 'Concurrency must be 1..20')
|
|
check(1 <= args.rounds <= 3 and 1 <= args.waves <= 5, 'Invalid rounds/waves')
|
|
check(0.1 <= args.upload_seconds <= 5, 'Pacing must be 0.1..5 seconds')
|
|
report = {'status': 'running', 'cases': [], 'settings': vars(args),
|
|
'method': 'Fresh API restart per case, exactly 2 verified warm-up uploads per replica via internal API addresses; temporary docker-exec client exits before measurement. Measured uploads use public Nginx. Two paced waves by default; all bodies are incomplete at the upload barrier, direct completes start at a second barrier. Full workflow latency includes grant/PUT/complete. External per-API-process RSS sampling at 50 ms; cgroup memory recorded separately. Independent replica peaks are not summed. No retries. Download verification is outside the measurement window.'}
|
|
output = ROOT / 'evidence/direct-upload-load.json'
|
|
def save(): output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
probes = []
|
|
try:
|
|
readiness = Client('http://127.0.0.1:8080', timeout=5)
|
|
deadline = time.monotonic() + 90
|
|
routed = set()
|
|
while time.monotonic() < deadline:
|
|
try: routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id'])
|
|
except Failure: pass
|
|
if len(routed) == 2: break
|
|
time.sleep(1)
|
|
check(len(routed) == 2, 'Both API replicas must be ready')
|
|
with fixture() as (client, owner, reader, chat, _):
|
|
containers = docker('compose', 'ps', '-q', 'api').split()
|
|
check(len(containers) == 2, 'Expected two replicas')
|
|
for container in containers:
|
|
flag = docker('exec', container, 'node', '-p', "process.env.UPLOAD_MEMORY_PROFILE || '0'").strip()
|
|
check(flag == '0', 'Disable detailed profiling for the load comparison')
|
|
cases = [(r + 1, side, concurrency, mode) for r in range(args.rounds)
|
|
for side in args.sizes for concurrency in args.concurrency
|
|
for mode in (['proxy', 'direct'] if r % 2 == 0 else ['direct', 'proxy'])]
|
|
for round_number, side, concurrency, mode in cases:
|
|
image = metrics.png_image(side) + metrics.chunk(b'IEND', b'')
|
|
digest = hashlib.sha256(image).hexdigest()
|
|
for probe in probes: probe.close()
|
|
probes.clear()
|
|
restarted = benchmark.restart_api(client, containers)
|
|
for container in containers:
|
|
probe = metrics.MemoryProbe(container)
|
|
probes.append(probe)
|
|
probe.ready = probe.receive('ready')
|
|
target_rows = docker('inspect', '--format', '{{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}', *containers).splitlines()
|
|
targets = [{'instance': row.split()[0], 'address': row.split()[1]} for row in target_rows]
|
|
source = (ROOT / 'test/warm-upload-replicas.mjs').read_text(encoding='utf-8')
|
|
warmup = json.loads(docker('exec', '-i', containers[0], 'node', '--input-type=module', '-e', source,
|
|
input=json.dumps({'targets': targets, 'owner': owner, 'reader': reader,
|
|
'chat': chat, 'imageBase64': base64.b64encode(image).decode(), 'mode': mode})))
|
|
check(warmup['counts'] == {p.ready['instance']: 2 for p in probes}, 'Unequal per-replica warm-up')
|
|
identities = docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers)
|
|
workers = [Client(client.base, timeout=60) for _ in range(concurrency)]
|
|
case = {'round': round_number, 'sidePixels': side, 'fileBytes': len(image), 'mode': mode,
|
|
'concurrency': concurrency, 'apiStatesAfterRestart': restarted,
|
|
'warmupCounts': warmup['counts'], 'waves': [], 'verifiedDownloads': 0}
|
|
report['cases'].append(case)
|
|
results = []
|
|
for probe in probes: probe.command('begin', 'begin')
|
|
started = time.monotonic()
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
|
for wave in range(args.waves):
|
|
gate = threading.Barrier(concurrency + 1)
|
|
complete_gate = threading.Barrier(concurrency)
|
|
futures = [pool.submit(upload, worker, owner, chat, image, digest, mode, gate, complete_gate, args.upload_seconds)
|
|
for worker in workers]
|
|
overlapped = False
|
|
try:
|
|
gate.wait(timeout=60)
|
|
overlapped = True
|
|
except threading.BrokenBarrierError: pass
|
|
wave_results = [future.result() for future in futures]
|
|
results.extend(wave_results)
|
|
case['waves'].append({'wave': wave + 1,
|
|
'incompleteBodiesAtBarrier': concurrency if overlapped else 0,
|
|
'attempts': [{k: v for k, v in result.items() if k != 'attachment'} for result in wave_results]})
|
|
finally:
|
|
case['elapsedSeconds'] = time.monotonic() - started
|
|
case['memory'] = [{'instance': p.ready['instance'], **p.command('end', 'summary')} for p in probes]
|
|
case['successfulUploads'] = sum(result['ok'] for result in results)
|
|
case['failedUploads'] = len(results) - case['successfulUploads']
|
|
case['successfulUploadsPerSecond'] = case['successfulUploads'] / case['elapsedSeconds']
|
|
case['instances'] = {name: sum(r['ok'] and r.get('instance') == name for r in results)
|
|
for name in sorted({r['instance'] for r in results if r['ok']})}
|
|
latencies = sorted(result['elapsedSeconds'] for result in results)
|
|
case['latencySeconds'] = {'median': statistics.median(latencies), 'max': max(latencies)}
|
|
case['apiIdentityUnchanged'] = identities == docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers)
|
|
save()
|
|
check(case['apiIdentityUnchanged'], 'API restarted during load')
|
|
for sample in case['memory']:
|
|
check(sample['samples'] >= 2, 'Insufficient RSS samples')
|
|
check(sample['oomDelta'] == sample['oomKillDelta'] == 0, 'API OOM; stopping further load')
|
|
for result in results:
|
|
if result['ok']:
|
|
verify(client, reader, result['attachment'], image)
|
|
case['verifiedDownloads'] += 1
|
|
check(len({r['attachment']['id'] for r in results if r['ok']}) == case['successfulUploads'], 'Duplicate attachment IDs')
|
|
if concurrency >= 5 and case['failedUploads'] == 0:
|
|
check(len(case['instances']) == 2, 'Concurrent work did not reach both replicas')
|
|
try:
|
|
case['apiReadyAfterCase'] = client.call('GET', '/health/ready')['status'] == 200
|
|
except Failure:
|
|
case['apiReadyAfterCase'] = False
|
|
save()
|
|
peaks = ', '.join(f"{m['sampledPeakRssBytes'] / 2**20:.1f}" for m in case['memory'])
|
|
print(f"DONE: round {round_number}, {mode}, {side}px, N={concurrency}, {case['successfulUploads']}/{len(results)}, RSS MiB=[{peaks}]", flush=True)
|
|
for probe in probes: probe.close()
|
|
probes.clear()
|
|
report['fixturesRemoved'] = True
|
|
report['status'] = 'completed_with_errors' if any(c['failedUploads'] or not c['apiReadyAfterCase'] for c in report['cases']) else 'passed'
|
|
return report['status'] == 'passed'
|
|
except BaseException as error:
|
|
report['status'] = 'failed'
|
|
report['errorType'] = type(error).__name__
|
|
if isinstance(error, Failure): report['failure'] = str(error)
|
|
raise
|
|
finally:
|
|
for probe in probes: probe.close()
|
|
save()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
sys.exit(0 if main() else 1)
|
|
except (Failure, ValueError, OSError, subprocess.SubprocessError) as error:
|
|
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
|
|
sys.exit(1)
|