forked from mtusi2026/mtusi2026lab4
feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
This commit is contained in:
@@ -0,0 +1,281 @@
|
||||
"""Concurrent raw PNG uploads through Nginx with per-API RSS measurements. Python 3.9+."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
from queue import Queue, Empty
|
||||
import random
|
||||
import ssl
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, shape
|
||||
|
||||
|
||||
class UploadFailure(Failure):
|
||||
def __init__(self, response):
|
||||
super().__init__(f'Concurrent upload returned {response.status}, expected 201')
|
||||
self.details = {'status': response.status,
|
||||
'instance': response.getheader('X-Instance-Id'),
|
||||
'requestId': response.getheader('X-Request-Id')}
|
||||
|
||||
|
||||
def docker(*args, **kwargs):
|
||||
return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
||||
text=True, check=True, timeout=90, **kwargs).stdout
|
||||
|
||||
|
||||
class MemoryProbe:
|
||||
def __init__(self, container):
|
||||
self.container = container
|
||||
self.messages = Queue()
|
||||
source = (ROOT / 'test/upload-memory-probe.mjs').read_text(encoding='utf-8')
|
||||
self.process = subprocess.Popen(
|
||||
['docker', 'exec', '-i', container, 'node', '--input-type=module', '-e', source],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True, bufsize=1,
|
||||
)
|
||||
threading.Thread(target=self.read, daemon=True).start()
|
||||
|
||||
def read(self):
|
||||
try:
|
||||
for line in self.process.stdout:
|
||||
self.messages.put(json.loads(line))
|
||||
except (ValueError, OSError):
|
||||
self.messages.put({'event': 'error'})
|
||||
finally:
|
||||
self.messages.put({'event': 'closed'})
|
||||
|
||||
def receive(self, expected):
|
||||
try:
|
||||
value = self.messages.get(timeout=45)
|
||||
except Empty:
|
||||
raise Failure('API memory probe timed out') from None
|
||||
check(value.get('event') == expected, 'API memory probe failed: ' + value.get('event', 'unknown'))
|
||||
return value
|
||||
|
||||
def command(self, command, response):
|
||||
self.process.stdin.write(command + '\n')
|
||||
self.process.stdin.flush()
|
||||
return self.receive(response)
|
||||
|
||||
def close(self):
|
||||
if self.process.poll() is None:
|
||||
try:
|
||||
self.process.stdin.write('quit\n')
|
||||
self.process.stdin.flush()
|
||||
self.process.stdin.close()
|
||||
self.process.wait(timeout=15)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
self.process.kill()
|
||||
self.process.wait(timeout=10)
|
||||
self.process.stdout.close()
|
||||
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||||
|
||||
|
||||
def png_image(side):
|
||||
# Deterministic noise is poorly compressible: the encoded file really occupies RAM.
|
||||
rng = random.Random(side)
|
||||
rows = b''.join(b'\0' + rng.randbytes(side * 3) for _ in range(side))
|
||||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0))
|
||||
+ chunk(b'IDAT', zlib.compress(rows, level=1)))
|
||||
|
||||
|
||||
def upload(base, actor, chat, body, filename, gate, duration, timeout):
|
||||
target = urlsplit(base + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename)
|
||||
connection_type = http.client.HTTPSConnection if target.scheme == 'https' else http.client.HTTPConnection
|
||||
kwargs = {'context': ssl.create_default_context()} if target.scheme == 'https' else {}
|
||||
connection = connection_type(target.hostname, target.port, timeout=timeout, **kwargs)
|
||||
try:
|
||||
connection.putrequest('POST', target.path + '?' + target.query)
|
||||
connection.putheader('Authorization', 'Bearer ' + actor['token'])
|
||||
connection.putheader('Content-Type', 'image/png')
|
||||
connection.putheader('Content-Length', str(len(body)))
|
||||
connection.endheaders()
|
||||
first = min(65536, len(body) - 1)
|
||||
connection.send(body[:first])
|
||||
# Every connection has sent headers and an incomplete body before any may finish.
|
||||
gate.wait(timeout=timeout)
|
||||
started = time.monotonic()
|
||||
for offset in range(first, len(body), 65536):
|
||||
end = min(offset + 65536, len(body))
|
||||
deadline = started + duration * (end - first) / (len(body) - first)
|
||||
time.sleep(max(0, deadline - time.monotonic()))
|
||||
connection.send(body[offset:end])
|
||||
response = connection.getresponse()
|
||||
data = response.read(1024 * 1024 + 1)
|
||||
if response.status != 201:
|
||||
raise UploadFailure(response)
|
||||
check(len(data) <= 1024 * 1024, 'Upload response unexpectedly large')
|
||||
check(response.getheader('Content-Type', '').startswith('application/json'), 'Upload response is not JSON')
|
||||
check(response.getheader('X-Request-Id'), 'Upload lacks request identifier')
|
||||
instance = response.getheader('X-Instance-Id')
|
||||
check(instance, 'Upload lacks instance identifier')
|
||||
attachment = shape(json.loads(data), 'Attachment')
|
||||
check(attachment['status'] == 'ready', 'Concurrent attachment is not ready')
|
||||
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == actor['id'], 'Incorrect attachment ownership')
|
||||
check(attachment['filename'] == filename and attachment['content_type'] == 'image/png', 'Incorrect attachment metadata')
|
||||
digest = hashlib.sha256(body).hexdigest()
|
||||
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(body), 'Upload bytes/metadata mismatch')
|
||||
return {'attachment': attachment, 'instance': instance, 'elapsedSeconds': time.monotonic() - started,
|
||||
'sha256': digest, 'sizeBytes': len(body)}
|
||||
except BaseException:
|
||||
gate.abort()
|
||||
raise
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--concurrency', type=int, default=10)
|
||||
parser.add_argument('--rounds', type=int, default=2)
|
||||
parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792], metavar='SIDE_PX')
|
||||
parser.add_argument('--upload-seconds', type=float, default=2)
|
||||
parser.add_argument('--timeout', type=float, default=60)
|
||||
parser.add_argument('--max-rss-mib', type=float, help='Optional assertion budget per API process, not a container resource limit')
|
||||
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/concurrent-uploads.json')
|
||||
args = parser.parse_args()
|
||||
check(10 <= args.concurrency <= 64, '--concurrency must be between 10 and 64')
|
||||
check(1 <= args.rounds <= 10, '--rounds must be between 1 and 10')
|
||||
check(all(64 <= side <= 1800 for side in args.sizes), '--sizes must be between 64 and 1800 pixels')
|
||||
check(0.5 <= args.upload_seconds <= 10 and args.timeout > args.upload_seconds + 5, 'Invalid upload timing')
|
||||
check(args.max_rss_mib is None or args.max_rss_mib > 0, 'RSS budget must be positive')
|
||||
client = Client(args.base_url, timeout=args.timeout)
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
run_id = str(uuid.uuid4())
|
||||
report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
|
||||
'concurrency': args.concurrency, 'roundsPerSize': args.rounds,
|
||||
'uploadSeconds': args.upload_seconds, 'maxRssMiB': args.max_rss_mib,
|
||||
'memoryMethod': 'API /proc VmRSS sampled every 50 ms; VmHWM is lifetime, not reset; probe RSS excluded',
|
||||
'waves': []}
|
||||
actors, probes, chat = [], [], None
|
||||
try:
|
||||
containers = docker('compose', 'ps', '-q', 'api').split()
|
||||
check(len(containers) >= 2, 'Expected at least two running API replicas')
|
||||
before = {state['Id']: (state['State']['StartedAt'], state['RestartCount'])
|
||||
for state in json.loads(docker('inspect', *containers))}
|
||||
for container in containers:
|
||||
probe = MemoryProbe(container)
|
||||
probes.append(probe)
|
||||
probe.ready = probe.receive('ready')
|
||||
owner = client.user('cu' + run_id[:8] + 'a')
|
||||
actors.append(owner)
|
||||
reader = client.user('cu' + run_id[:8] + 'b')
|
||||
actors.append(reader)
|
||||
chat = client.call('POST', '/api/v1/chats', 201, actor=owner,
|
||||
body={'title': 'concurrency:' + run_id, 'member_ids': [actor['id'] for actor in actors]},
|
||||
schema='Chat')['json']
|
||||
for side in args.sizes:
|
||||
image = png_image(side)
|
||||
for number in range(1, args.rounds + 1):
|
||||
bodies = [image + chunk(b'tEXt', f'fixture\0{run_id}-{side}-{number}-{i}'.encode()) + chunk(b'IEND', b'')
|
||||
for i in range(args.concurrency)]
|
||||
check(all(len(body) <= 10485760 for body in bodies), 'Generated PNG exceeds 10 MiB')
|
||||
wave = {'sidePixels': side, 'round': number, 'fileBytes': len(bodies[0]),
|
||||
'totalBytes': sum(map(len, bodies)), 'incompleteUploadsAtBarrier': 0,
|
||||
'successfulUploads': 0, 'verifiedDownloads': 0, 'memory': []}
|
||||
report['waves'].append(wave)
|
||||
for probe in probes:
|
||||
probe.command('begin', 'begin')
|
||||
gate = threading.Barrier(args.concurrency + 1)
|
||||
started = time.monotonic()
|
||||
futures = []
|
||||
print(f'Uploading: {args.concurrency} x {len(bodies[0]) / 1048576:.2f} MiB, round {number}', flush=True)
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
futures = [pool.submit(upload, client.base, owner, chat, body,
|
||||
f'{side}-{number}-{i}.png', gate, args.upload_seconds, args.timeout)
|
||||
for i, body in enumerate(bodies)]
|
||||
gate.wait(timeout=args.timeout)
|
||||
wave['incompleteUploadsAtBarrier'] = args.concurrency
|
||||
results = [future.result() for future in futures]
|
||||
finally:
|
||||
wave['uploadWallSeconds'] = time.monotonic() - started
|
||||
memory = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes]
|
||||
wave['memory'] = memory
|
||||
wave['successfulUploads'] = sum(future.done() and not future.cancelled() and future.exception() is None for future in futures)
|
||||
wave['failedUploads'] = [
|
||||
{'index': index, 'errorType': type(future.exception()).__name__,
|
||||
**getattr(future.exception(), 'details', {})}
|
||||
for index, future in enumerate(futures)
|
||||
if future.done() and not future.cancelled() and future.exception() is not None
|
||||
]
|
||||
wave['instances'] = {name: sum(r['instance'] == name for r in results) for name in sorted({r['instance'] for r in results})}
|
||||
wave['latencySeconds'] = {'min': min(r['elapsedSeconds'] for r in results), 'max': max(r['elapsedSeconds'] for r in results)}
|
||||
check(len({r['attachment']['id'] for r in results}) == args.concurrency, 'Attachment IDs collide')
|
||||
check(len(wave['instances']) >= 2, 'Concurrent uploads did not reach both replicas')
|
||||
check(set(wave['instances']).issubset({probe.ready['instance'] for probe in probes}), 'Unmeasured API replica served uploads')
|
||||
for sample in memory:
|
||||
check(sample['samples'] >= 2, 'Insufficient memory samples during upload')
|
||||
check(sample['oomDelta'] == 0 and sample['oomKillDelta'] == 0, 'API cgroup reported OOM during upload')
|
||||
if args.max_rss_mib is not None:
|
||||
check(sample['sampledPeakRssBytes'] <= args.max_rss_mib * 1048576, 'API sampled RSS exceeded assertion budget')
|
||||
for result in results:
|
||||
path = '/api/v1/attachments/' + result['attachment']['id']
|
||||
meta = client.call('GET', path, actor=reader, schema='Attachment')['json']
|
||||
check(meta == result['attachment'], 'Concurrent metadata changed')
|
||||
data = client.call('GET', path + '/content', actor=reader)['bytes']
|
||||
check(len(data) == result['sizeBytes'] and hashlib.sha256(data).hexdigest() == result['sha256'], 'Concurrent download checksum mismatch')
|
||||
wave['verifiedDownloads'] += 1
|
||||
peaks = ', '.join(f"{s['instance']}: {s['sampledPeakRssBytes'] / 1048576:.1f} MiB" for s in memory)
|
||||
print('PASS: all uploads/downloads verified; sampled API RSS peaks: ' + peaks, flush=True)
|
||||
after = json.loads(docker('inspect', *containers))
|
||||
check(all(before[state['Id']] == (state['State']['StartedAt'], state['RestartCount'])
|
||||
and state['State']['Running'] and not state['State']['OOMKilled'] for state in after), 'API restarted or was OOM-killed')
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
report['status'] = 'passed'
|
||||
except BaseException as error:
|
||||
report['status'] = 'failed'
|
||||
report['errorType'] = type(error).__name__
|
||||
if isinstance(error, Failure):
|
||||
report['failure'] = str(error)
|
||||
raise
|
||||
finally:
|
||||
for probe in probes:
|
||||
if hasattr(probe, 'process'):
|
||||
try:
|
||||
probe.close()
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
report['probeCloseFailed'] = True
|
||||
try:
|
||||
source = (ROOT / 'test/concurrent-cleanup.mjs').read_text(encoding='utf-8')
|
||||
payload = json.dumps({'runId': run_id, 'chatId': chat['id'] if chat else None,
|
||||
'userIds': [actor['id'] for actor in actors]})
|
||||
docker('compose', 'exec', '-T', '--index', '1', 'api', 'node', '--input-type=module', '-e', source, input=payload)
|
||||
report['fixturesRemoved'] = True
|
||||
except BaseException:
|
||||
report['fixturesRemoved'] = False
|
||||
if report['status'] == 'passed':
|
||||
report['status'] = 'failed'
|
||||
raise Failure('Concurrent fixtures cleanup failed') from None
|
||||
finally:
|
||||
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
||||
print('PASS: concurrent uploads, byte integrity, memory sampling and no API restarts/OOM', flush=True)
|
||||
print('Report: ' + str(args.output))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (Failure, ValueError, OSError, subprocess.SubprocessError, threading.BrokenBarrierError) as error:
|
||||
# Never include subprocess arguments or request headers: they can contain credentials.
|
||||
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user