Files
mtusi2026lab4/test/direct-upload-benchmark.py
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

168 lines
9.6 KiB
Python

"""Compare the entire proxy/direct upload workflows, including complete validation."""
import argparse
import importlib.util
import datetime
import json
import subprocess
import sys
import time
from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify
spec = importlib.util.spec_from_file_location('upload_metrics', ROOT / 'test/concurrent-uploads.py')
metrics = importlib.util.module_from_spec(spec)
spec.loader.exec_module(metrics)
def collect_profiles(case, since, repeats):
records = {}
for line in docker('compose', 'logs', '--no-color', '--since', since, 'api').splitlines():
if 'UPLOAD_MEMORY_PROFILE ' in line:
record = json.loads(line.split('UPLOAD_MEMORY_PROFILE ', 1)[1])
records[record['requestId']] = record
case['profiles'] = [records[request_id] for request_id in case['requestIds'] if request_id in records]
check(len(case['profiles']) == repeats, 'Missing opt-in API stage profiles')
for record in case['profiles']:
check(record['mode'] == case['mode'] and record['status'] == 201 and record['finished'], 'Invalid stage profile')
names = {stage['name'] for stage in record['stages']}
check({'sharp_metadata', 'sharp_decode_stats', 'serialize_response'} <= names, 'Async stage context was lost')
if case['mode'] == 'direct':
check({'s3_stream_sha256_to_file', 'sha256_finalize', 's3_copy'} <= names, 'Missing direct profile stages')
def restart_api(client, containers):
def states():
return docker('inspect', '--format', '{{.State.Health.Status}} {{.State.StartedAt}}', *containers).splitlines()
before = states()
docker('compose', 'restart', 'api')
deadline = time.monotonic() + 120
routed = set()
while time.monotonic() < deadline:
current = states()
if len(current) == 2 and all(line.startswith('healthy ') for line in current):
check(all(old != new for old, new in zip(before, current)), 'API did not restart')
try:
response = client.call('GET', '/health/ready')
routed.add(response['headers']['X-Instance-Id'])
if len(routed) == 2:
return current
except Failure:
pass
time.sleep(1)
raise Failure('API did not become ready after restart')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792])
parser.add_argument('--repeats', type=int, default=3)
parser.add_argument('--restart-api', action='store_true', help='Restart both API replicas before each measured case')
parser.add_argument('--rounds', type=int, default=1, help='Reverse workflow order on alternate rounds')
parser.add_argument('--profile', action='store_true', help='Collect opt-in API stage profiles and RSS after a five-second idle period')
args = parser.parse_args()
check(len(set(args.sizes)) >= 2 and all(64 <= side <= 1800 for side in args.sizes), 'Choose at least two image sides between 64 and 1800')
check(1 <= args.repeats <= 10, 'Repeats must be between 1 and 10')
check(1 <= args.rounds <= 5, 'Rounds must be between 1 and 5')
warmups = 4 if args.restart_api else 1
report = {'status': 'failed', 'repeats': args.repeats, 'rounds': args.rounds,
'restartApiBeforeCase': args.restart_api, 'warmupUploadsPerCase': warmups, 'cases': [],
'method': 'One upload at a time; measured direct flow includes grant, PUT, and complete with full image validation. Download verification is outside the measurement window. RSS sampling 50 ms; network counters sum non-loopback API interfaces, including DB/S3/background health traffic. Probe RSS excluded.'}
probes = []
log_start = datetime.datetime.now(datetime.timezone.utc).isoformat()
try:
readiness = Client('http://127.0.0.1:8080', timeout=5)
deadline = time.monotonic() + 90
routed = set()
while time.monotonic() < deadline:
try:
routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id'])
if len(routed) == 2: break
except Failure:
pass
time.sleep(1)
check(len(routed) == 2, 'Both API replicas must be ready through Nginx')
with fixture() as (client, owner, reader, chat, _):
containers = docker('compose', 'ps', '-q', 'api').split()
check(len(containers) == 2, 'Expected two API replicas')
cases = [(round_index + 1, side, mode) for round_index in range(args.rounds)
for side in args.sizes
for mode in (['proxy', 'direct'] if round_index % 2 == 0 else ['direct', 'proxy'])]
for round_number, side, mode in cases:
image = metrics.png_image(side) + metrics.chunk(b'IEND', b'')
for probe in probes: probe.close()
probes.clear()
restarted = restart_api(client, containers) if args.restart_api else None
for container in containers:
probe = metrics.MemoryProbe(container)
probes.append(probe)
probe.ready = probe.receive('ready')
def upload():
if mode == 'direct':
cap = grant(client, owner, chat, image)
put(client, cap, image)
response = complete(client, owner, cap)
else:
response = client.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=benchmark.png", 201,
actor=owner, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment')
return response['json'], response['headers']['X-Instance-Id'], response['headers']['X-Request-Id']
# Warm the selected workflow before recording it. Warm-up failures fail the run.
warmed = set()
for warmup_index in range(warmups):
# Vary request parity so least_conn does not pin every complete
# to the same replica in the four-request direct workflow.
if args.restart_api and warmup_index % 2:
client.call('GET', '/health/live')
attachment, instance, _ = upload()
warmed.add(instance)
verify(client, reader, attachment, image)
if args.restart_api:
check(warmed == {probe.ready['instance'] for probe in probes}, 'Warm-up did not cover both API replicas')
for probe in probes: probe.command('begin', 'begin')
started = time.monotonic()
attachments = []
request_ids = []
try:
for _ in range(args.repeats):
attachment, _, request_id = upload()
attachments.append(attachment)
request_ids.append(request_id)
finally:
elapsed = time.monotonic() - started
samples = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes]
report['cases'].append({'round': round_number, 'mode': mode, 'sidePixels': side, 'fileBytes': len(image),
'apiStatesAfterRestart': restarted, 'warmupInstances': sorted(warmed),
'requestIds': request_ids,
'successfulUploads': len(attachments), 'elapsedSeconds': elapsed,
'memory': samples, 'apiRxBytes': sum(m['networkRxBytes'] for m in samples),
'apiTxBytes': sum(m['networkTxBytes'] for m in samples)})
for sample in samples:
check(sample['samples'] >= 2 and sample['oomDelta'] == sample['oomKillDelta'] == 0, 'Invalid memory samples or OOM')
if args.profile:
time.sleep(5)
report['cases'][-1]['idleAfterFiveSeconds'] = [
{'instance': probe.ready['instance'], **probe.command('snapshot', 'snapshot')} for probe in probes]
collect_profiles(report['cases'][-1], log_start, args.repeats)
for attachment in attachments: verify(client, reader, attachment, image)
print(f"PASS: round {round_number}, {mode}, {side}px, {len(image)} bytes, {len(attachments)} uploads; API RX/TX={sum(m['networkRxBytes'] for m in samples)}/{sum(m['networkTxBytes'] for m in samples)}", flush=True)
for probe in probes: probe.close()
probes.clear()
report['fixturesRemoved'] = True
report['status'] = 'passed'
except BaseException as error:
report['errorType'] = type(error).__name__
if isinstance(error, Failure): report['failure'] = str(error)
raise
finally:
for probe in probes: probe.close()
filename = 'direct-upload-memory-profile.json' if args.profile else (
'direct-upload-benchmark-restarted.json' if args.restart_api else 'direct-upload-benchmark.json')
(ROOT / 'evidence' / filename).write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError, subprocess.SubprocessError) as error:
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
sys.exit(1)