Files
mtusi2026lab4/test/direct-upload-load.py
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

238 lines
14 KiB
Python

"""Compare proxy/direct uploads under synchronized load; measure the main API processes."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import http.client
import importlib.util
import json
import statistics
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
from direct_client import ROOT, Client, Failure, check, docker, fixture, grant, put, complete, verify
spec = importlib.util.spec_from_file_location('direct_benchmark', ROOT / 'test/direct-upload-benchmark.py')
benchmark = importlib.util.module_from_spec(spec)
spec.loader.exec_module(benchmark)
metrics = benchmark.metrics
class HttpFailure(Failure):
def __init__(self, stage, status, headers):
super().__init__(f'{stage} returned HTTP {status}')
self.details = {'stage': stage, 'status': status,
'instance': headers.get('X-Instance-Id'), 'requestId': headers.get('X-Request-Id')}
def paced_request(method, url, headers, image, gate, duration):
target = urlsplit(url)
check(target.scheme == 'http', 'This benchmark targets the local HTTP stack')
connection = http.client.HTTPConnection(target.hostname, target.port, timeout=60)
try:
connection.putrequest(method, target.path + ('?' + target.query if target.query else ''))
for name, value in headers.items(): connection.putheader(name, value)
connection.endheaders()
first = min(65536, len(image) - 1)
connection.send(image[:first])
gate.wait(timeout=60)
started = time.monotonic()
for offset in range(first, len(image), 65536):
end = min(offset + 65536, len(image))
deadline = started + duration * (end - first) / (len(image) - first)
time.sleep(max(0, deadline - time.monotonic()))
connection.send(image[offset:end])
response = connection.getresponse()
body = response.read(1024 * 1024 + 1)
check(len(body) <= 1024 * 1024, 'Unexpectedly large response')
return response.status, response.headers, body
finally:
connection.close()
def error_result(error, stage):
# Never serialize exceptions containing a signed URL or request headers.
return {'ok': False, 'stage': stage, 'errorType': type(error).__name__,
**getattr(error, 'details', {})}
def upload(client, owner, chat, image, digest, mode, gate, completion_gate, duration):
started = time.monotonic()
error = None
attachment = None
instance = None
stage = 'grant' if mode == 'direct' else 'proxy'
try:
if mode == 'direct':
capability = grant(client, owner, chat, image)
stage = 'put'
status, headers, _ = paced_request('PUT', capability['url'], capability['headers'], image, gate, duration)
if status != 200: raise HttpFailure(stage, status, headers)
else:
status, headers, body = paced_request(
'POST', client.base + f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
{'Authorization': 'Bearer ' + owner['token'], 'Content-Type': 'image/png',
'Content-Length': str(len(image))}, image, gate, duration)
if status != 201: raise HttpFailure(stage, status, headers)
attachment = metrics.shape(json.loads(body), 'Attachment')
instance = headers.get('X-Instance-Id')
except Exception as caught:
error = error_result(caught, stage)
gate.abort()
if mode == 'direct':
# All workers arrive even if their PUT failed; successful PUTs can still complete.
try:
completion_gate.wait(timeout=65)
if error is None:
stage = 'complete'
response = complete(client, owner, capability, [201, 400, 401, 404, 408, 409, 413, 415, 429, 500, 503])
if response['status'] != 201: raise HttpFailure(stage, response['status'], response['headers'])
attachment = metrics.shape(json.loads(response['bytes']), 'Attachment')
instance = response['headers'].get('X-Instance-Id')
except Exception as caught:
if error is None: error = error_result(caught, stage)
if error is None:
try:
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(image), 'Incorrect checksum/size')
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == owner['id'], 'Incorrect ownership')
check(bool(instance), 'Missing instance ID')
except Exception as caught:
error = error_result(caught, 'validate_response')
return {**(error or {'ok': True, 'attachment': attachment, 'instance': instance}),
'elapsedSeconds': time.monotonic() - started}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--sizes', type=int, nargs='+', default=[1792])
parser.add_argument('--concurrency', type=int, nargs='+', default=[1, 5, 10])
parser.add_argument('--rounds', type=int, default=2)
parser.add_argument('--waves', type=int, default=2)
parser.add_argument('--upload-seconds', type=float, default=2)
args = parser.parse_args()
check(all(64 <= n <= 1800 for n in args.sizes), 'Invalid image size')
check(all(1 <= n <= 20 for n in args.concurrency), 'Concurrency must be 1..20')
check(1 <= args.rounds <= 3 and 1 <= args.waves <= 5, 'Invalid rounds/waves')
check(0.1 <= args.upload_seconds <= 5, 'Pacing must be 0.1..5 seconds')
report = {'status': 'running', 'cases': [], 'settings': vars(args),
'method': 'Fresh API restart per case, exactly 2 verified warm-up uploads per replica via internal API addresses; temporary docker-exec client exits before measurement. Measured uploads use public Nginx. Two paced waves by default; all bodies are incomplete at the upload barrier, direct completes start at a second barrier. Full workflow latency includes grant/PUT/complete. External per-API-process RSS sampling at 50 ms; cgroup memory recorded separately. Independent replica peaks are not summed. No retries. Download verification is outside the measurement window.'}
output = ROOT / 'evidence/direct-upload-load.json'
def save(): output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
probes = []
try:
readiness = Client('http://127.0.0.1:8080', timeout=5)
deadline = time.monotonic() + 90
routed = set()
while time.monotonic() < deadline:
try: routed.add(readiness.call('GET', '/health/ready')['headers']['X-Instance-Id'])
except Failure: pass
if len(routed) == 2: break
time.sleep(1)
check(len(routed) == 2, 'Both API replicas must be ready')
with fixture() as (client, owner, reader, chat, _):
containers = docker('compose', 'ps', '-q', 'api').split()
check(len(containers) == 2, 'Expected two replicas')
for container in containers:
flag = docker('exec', container, 'node', '-p', "process.env.UPLOAD_MEMORY_PROFILE || '0'").strip()
check(flag == '0', 'Disable detailed profiling for the load comparison')
cases = [(r + 1, side, concurrency, mode) for r in range(args.rounds)
for side in args.sizes for concurrency in args.concurrency
for mode in (['proxy', 'direct'] if r % 2 == 0 else ['direct', 'proxy'])]
for round_number, side, concurrency, mode in cases:
image = metrics.png_image(side) + metrics.chunk(b'IEND', b'')
digest = hashlib.sha256(image).hexdigest()
for probe in probes: probe.close()
probes.clear()
restarted = benchmark.restart_api(client, containers)
for container in containers:
probe = metrics.MemoryProbe(container)
probes.append(probe)
probe.ready = probe.receive('ready')
target_rows = docker('inspect', '--format', '{{.Config.Hostname}} {{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}', *containers).splitlines()
targets = [{'instance': row.split()[0], 'address': row.split()[1]} for row in target_rows]
source = (ROOT / 'test/warm-upload-replicas.mjs').read_text(encoding='utf-8')
warmup = json.loads(docker('exec', '-i', containers[0], 'node', '--input-type=module', '-e', source,
input=json.dumps({'targets': targets, 'owner': owner, 'reader': reader,
'chat': chat, 'imageBase64': base64.b64encode(image).decode(), 'mode': mode})))
check(warmup['counts'] == {p.ready['instance']: 2 for p in probes}, 'Unequal per-replica warm-up')
identities = docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers)
workers = [Client(client.base, timeout=60) for _ in range(concurrency)]
case = {'round': round_number, 'sidePixels': side, 'fileBytes': len(image), 'mode': mode,
'concurrency': concurrency, 'apiStatesAfterRestart': restarted,
'warmupCounts': warmup['counts'], 'waves': [], 'verifiedDownloads': 0}
report['cases'].append(case)
results = []
for probe in probes: probe.command('begin', 'begin')
started = time.monotonic()
try:
with ThreadPoolExecutor(max_workers=concurrency) as pool:
for wave in range(args.waves):
gate = threading.Barrier(concurrency + 1)
complete_gate = threading.Barrier(concurrency)
futures = [pool.submit(upload, worker, owner, chat, image, digest, mode, gate, complete_gate, args.upload_seconds)
for worker in workers]
overlapped = False
try:
gate.wait(timeout=60)
overlapped = True
except threading.BrokenBarrierError: pass
wave_results = [future.result() for future in futures]
results.extend(wave_results)
case['waves'].append({'wave': wave + 1,
'incompleteBodiesAtBarrier': concurrency if overlapped else 0,
'attempts': [{k: v for k, v in result.items() if k != 'attachment'} for result in wave_results]})
finally:
case['elapsedSeconds'] = time.monotonic() - started
case['memory'] = [{'instance': p.ready['instance'], **p.command('end', 'summary')} for p in probes]
case['successfulUploads'] = sum(result['ok'] for result in results)
case['failedUploads'] = len(results) - case['successfulUploads']
case['successfulUploadsPerSecond'] = case['successfulUploads'] / case['elapsedSeconds']
case['instances'] = {name: sum(r['ok'] and r.get('instance') == name for r in results)
for name in sorted({r['instance'] for r in results if r['ok']})}
latencies = sorted(result['elapsedSeconds'] for result in results)
case['latencySeconds'] = {'median': statistics.median(latencies), 'max': max(latencies)}
case['apiIdentityUnchanged'] = identities == docker('inspect', '--format', '{{.State.StartedAt}} {{.RestartCount}} {{.State.OOMKilled}} {{.State.Running}}', *containers)
save()
check(case['apiIdentityUnchanged'], 'API restarted during load')
for sample in case['memory']:
check(sample['samples'] >= 2, 'Insufficient RSS samples')
check(sample['oomDelta'] == sample['oomKillDelta'] == 0, 'API OOM; stopping further load')
for result in results:
if result['ok']:
verify(client, reader, result['attachment'], image)
case['verifiedDownloads'] += 1
check(len({r['attachment']['id'] for r in results if r['ok']}) == case['successfulUploads'], 'Duplicate attachment IDs')
if concurrency >= 5 and case['failedUploads'] == 0:
check(len(case['instances']) == 2, 'Concurrent work did not reach both replicas')
try:
case['apiReadyAfterCase'] = client.call('GET', '/health/ready')['status'] == 200
except Failure:
case['apiReadyAfterCase'] = False
save()
peaks = ', '.join(f"{m['sampledPeakRssBytes'] / 2**20:.1f}" for m in case['memory'])
print(f"DONE: round {round_number}, {mode}, {side}px, N={concurrency}, {case['successfulUploads']}/{len(results)}, RSS MiB=[{peaks}]", flush=True)
for probe in probes: probe.close()
probes.clear()
report['fixturesRemoved'] = True
report['status'] = 'completed_with_errors' if any(c['failedUploads'] or not c['apiReadyAfterCase'] for c in report['cases']) else 'passed'
return report['status'] == 'passed'
except BaseException as error:
report['status'] = 'failed'
report['errorType'] = type(error).__name__
if isinstance(error, Failure): report['failure'] = str(error)
raise
finally:
for probe in probes: probe.close()
save()
if __name__ == '__main__':
try:
sys.exit(0 if main() else 1)
except (Failure, ValueError, OSError, subprocess.SubprocessError) as error:
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
sys.exit(1)