Files
mtusi2026lab4/test/concurrent-uploads.py
T
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

282 lines
15 KiB
Python

"""Concurrent raw PNG uploads through Nginx with per-API RSS measurements. Python 3.9+."""
import argparse
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import http.client
import json
from pathlib import Path
from queue import Queue, Empty
import random
import ssl
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import uuid
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check, shape
class UploadFailure(Failure):
def __init__(self, response):
super().__init__(f'Concurrent upload returned {response.status}, expected 201')
self.details = {'status': response.status,
'instance': response.getheader('X-Instance-Id'),
'requestId': response.getheader('X-Request-Id')}
def docker(*args, **kwargs):
return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
text=True, check=True, timeout=90, **kwargs).stdout
class MemoryProbe:
def __init__(self, container):
self.container = container
self.messages = Queue()
source = (ROOT / 'test/upload-memory-probe.mjs').read_text(encoding='utf-8')
self.process = subprocess.Popen(
['docker', 'exec', '-i', container, 'node', '--input-type=module', '-e', source],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, bufsize=1,
)
threading.Thread(target=self.read, daemon=True).start()
def read(self):
try:
for line in self.process.stdout:
self.messages.put(json.loads(line))
except (ValueError, OSError):
self.messages.put({'event': 'error'})
finally:
self.messages.put({'event': 'closed'})
def receive(self, expected):
try:
value = self.messages.get(timeout=45)
except Empty:
raise Failure('API memory probe timed out') from None
check(value.get('event') == expected, 'API memory probe failed: ' + value.get('event', 'unknown'))
return value
def command(self, command, response):
self.process.stdin.write(command + '\n')
self.process.stdin.flush()
return self.receive(response)
def close(self):
if self.process.poll() is None:
try:
self.process.stdin.write('quit\n')
self.process.stdin.flush()
self.process.stdin.close()
self.process.wait(timeout=15)
except (OSError, subprocess.TimeoutExpired):
self.process.kill()
self.process.wait(timeout=10)
self.process.stdout.close()
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
def png_image(side):
# Deterministic noise is poorly compressible: the encoded file really occupies RAM.
rng = random.Random(side)
rows = b''.join(b'\0' + rng.randbytes(side * 3) for _ in range(side))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(rows, level=1)))
def upload(base, actor, chat, body, filename, gate, duration, timeout):
target = urlsplit(base + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename)
connection_type = http.client.HTTPSConnection if target.scheme == 'https' else http.client.HTTPConnection
kwargs = {'context': ssl.create_default_context()} if target.scheme == 'https' else {}
connection = connection_type(target.hostname, target.port, timeout=timeout, **kwargs)
try:
connection.putrequest('POST', target.path + '?' + target.query)
connection.putheader('Authorization', 'Bearer ' + actor['token'])
connection.putheader('Content-Type', 'image/png')
connection.putheader('Content-Length', str(len(body)))
connection.endheaders()
first = min(65536, len(body) - 1)
connection.send(body[:first])
# Every connection has sent headers and an incomplete body before any may finish.
gate.wait(timeout=timeout)
started = time.monotonic()
for offset in range(first, len(body), 65536):
end = min(offset + 65536, len(body))
deadline = started + duration * (end - first) / (len(body) - first)
time.sleep(max(0, deadline - time.monotonic()))
connection.send(body[offset:end])
response = connection.getresponse()
data = response.read(1024 * 1024 + 1)
if response.status != 201:
raise UploadFailure(response)
check(len(data) <= 1024 * 1024, 'Upload response unexpectedly large')
check(response.getheader('Content-Type', '').startswith('application/json'), 'Upload response is not JSON')
check(response.getheader('X-Request-Id'), 'Upload lacks request identifier')
instance = response.getheader('X-Instance-Id')
check(instance, 'Upload lacks instance identifier')
attachment = shape(json.loads(data), 'Attachment')
check(attachment['status'] == 'ready', 'Concurrent attachment is not ready')
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == actor['id'], 'Incorrect attachment ownership')
check(attachment['filename'] == filename and attachment['content_type'] == 'image/png', 'Incorrect attachment metadata')
digest = hashlib.sha256(body).hexdigest()
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(body), 'Upload bytes/metadata mismatch')
return {'attachment': attachment, 'instance': instance, 'elapsedSeconds': time.monotonic() - started,
'sha256': digest, 'sizeBytes': len(body)}
except BaseException:
gate.abort()
raise
finally:
connection.close()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--rounds', type=int, default=2)
parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792], metavar='SIDE_PX')
parser.add_argument('--upload-seconds', type=float, default=2)
parser.add_argument('--timeout', type=float, default=60)
parser.add_argument('--max-rss-mib', type=float, help='Optional assertion budget per API process, not a container resource limit')
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/concurrent-uploads.json')
args = parser.parse_args()
check(10 <= args.concurrency <= 64, '--concurrency must be between 10 and 64')
check(1 <= args.rounds <= 10, '--rounds must be between 1 and 10')
check(all(64 <= side <= 1800 for side in args.sizes), '--sizes must be between 64 and 1800 pixels')
check(0.5 <= args.upload_seconds <= 10 and args.timeout > args.upload_seconds + 5, 'Invalid upload timing')
check(args.max_rss_mib is None or args.max_rss_mib > 0, 'RSS budget must be positive')
client = Client(args.base_url, timeout=args.timeout)
client.call('GET', '/health/ready', schema='Health')
run_id = str(uuid.uuid4())
report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
'concurrency': args.concurrency, 'roundsPerSize': args.rounds,
'uploadSeconds': args.upload_seconds, 'maxRssMiB': args.max_rss_mib,
'memoryMethod': 'API /proc VmRSS sampled every 50 ms; VmHWM is lifetime, not reset; probe RSS excluded',
'waves': []}
actors, probes, chat = [], [], None
try:
containers = docker('compose', 'ps', '-q', 'api').split()
check(len(containers) >= 2, 'Expected at least two running API replicas')
before = {state['Id']: (state['State']['StartedAt'], state['RestartCount'])
for state in json.loads(docker('inspect', *containers))}
for container in containers:
probe = MemoryProbe(container)
probes.append(probe)
probe.ready = probe.receive('ready')
owner = client.user('cu' + run_id[:8] + 'a')
actors.append(owner)
reader = client.user('cu' + run_id[:8] + 'b')
actors.append(reader)
chat = client.call('POST', '/api/v1/chats', 201, actor=owner,
body={'title': 'concurrency:' + run_id, 'member_ids': [actor['id'] for actor in actors]},
schema='Chat')['json']
for side in args.sizes:
image = png_image(side)
for number in range(1, args.rounds + 1):
bodies = [image + chunk(b'tEXt', f'fixture\0{run_id}-{side}-{number}-{i}'.encode()) + chunk(b'IEND', b'')
for i in range(args.concurrency)]
check(all(len(body) <= 10485760 for body in bodies), 'Generated PNG exceeds 10 MiB')
wave = {'sidePixels': side, 'round': number, 'fileBytes': len(bodies[0]),
'totalBytes': sum(map(len, bodies)), 'incompleteUploadsAtBarrier': 0,
'successfulUploads': 0, 'verifiedDownloads': 0, 'memory': []}
report['waves'].append(wave)
for probe in probes:
probe.command('begin', 'begin')
gate = threading.Barrier(args.concurrency + 1)
started = time.monotonic()
futures = []
print(f'Uploading: {args.concurrency} x {len(bodies[0]) / 1048576:.2f} MiB, round {number}', flush=True)
try:
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
futures = [pool.submit(upload, client.base, owner, chat, body,
f'{side}-{number}-{i}.png', gate, args.upload_seconds, args.timeout)
for i, body in enumerate(bodies)]
gate.wait(timeout=args.timeout)
wave['incompleteUploadsAtBarrier'] = args.concurrency
results = [future.result() for future in futures]
finally:
wave['uploadWallSeconds'] = time.monotonic() - started
memory = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes]
wave['memory'] = memory
wave['successfulUploads'] = sum(future.done() and not future.cancelled() and future.exception() is None for future in futures)
wave['failedUploads'] = [
{'index': index, 'errorType': type(future.exception()).__name__,
**getattr(future.exception(), 'details', {})}
for index, future in enumerate(futures)
if future.done() and not future.cancelled() and future.exception() is not None
]
wave['instances'] = {name: sum(r['instance'] == name for r in results) for name in sorted({r['instance'] for r in results})}
wave['latencySeconds'] = {'min': min(r['elapsedSeconds'] for r in results), 'max': max(r['elapsedSeconds'] for r in results)}
check(len({r['attachment']['id'] for r in results}) == args.concurrency, 'Attachment IDs collide')
check(len(wave['instances']) >= 2, 'Concurrent uploads did not reach both replicas')
check(set(wave['instances']).issubset({probe.ready['instance'] for probe in probes}), 'Unmeasured API replica served uploads')
for sample in memory:
check(sample['samples'] >= 2, 'Insufficient memory samples during upload')
check(sample['oomDelta'] == 0 and sample['oomKillDelta'] == 0, 'API cgroup reported OOM during upload')
if args.max_rss_mib is not None:
check(sample['sampledPeakRssBytes'] <= args.max_rss_mib * 1048576, 'API sampled RSS exceeded assertion budget')
for result in results:
path = '/api/v1/attachments/' + result['attachment']['id']
meta = client.call('GET', path, actor=reader, schema='Attachment')['json']
check(meta == result['attachment'], 'Concurrent metadata changed')
data = client.call('GET', path + '/content', actor=reader)['bytes']
check(len(data) == result['sizeBytes'] and hashlib.sha256(data).hexdigest() == result['sha256'], 'Concurrent download checksum mismatch')
wave['verifiedDownloads'] += 1
peaks = ', '.join(f"{s['instance']}: {s['sampledPeakRssBytes'] / 1048576:.1f} MiB" for s in memory)
print('PASS: all uploads/downloads verified; sampled API RSS peaks: ' + peaks, flush=True)
after = json.loads(docker('inspect', *containers))
check(all(before[state['Id']] == (state['State']['StartedAt'], state['RestartCount'])
and state['State']['Running'] and not state['State']['OOMKilled'] for state in after), 'API restarted or was OOM-killed')
client.call('GET', '/health/ready', schema='Health')
report['status'] = 'passed'
except BaseException as error:
report['status'] = 'failed'
report['errorType'] = type(error).__name__
if isinstance(error, Failure):
report['failure'] = str(error)
raise
finally:
for probe in probes:
if hasattr(probe, 'process'):
try:
probe.close()
except (OSError, subprocess.SubprocessError):
report['probeCloseFailed'] = True
try:
source = (ROOT / 'test/concurrent-cleanup.mjs').read_text(encoding='utf-8')
payload = json.dumps({'runId': run_id, 'chatId': chat['id'] if chat else None,
'userIds': [actor['id'] for actor in actors]})
docker('compose', 'exec', '-T', '--index', '1', 'api', 'node', '--input-type=module', '-e', source, input=payload)
report['fixturesRemoved'] = True
except BaseException:
report['fixturesRemoved'] = False
if report['status'] == 'passed':
report['status'] = 'failed'
raise Failure('Concurrent fixtures cleanup failed') from None
finally:
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print('PASS: concurrent uploads, byte integrity, memory sampling and no API restarts/OOM', flush=True)
print('Report: ' + str(args.output))
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError, subprocess.SubprocessError, threading.BrokenBarrierError) as error:
# Never include subprocess arguments or request headers: they can contain credentials.
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
sys.exit(1)