forked from mtusi2026/mtusi2026lab4
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
282 lines
15 KiB
Python
282 lines
15 KiB
Python
"""Concurrent raw PNG uploads through Nginx with per-API RSS measurements. Python 3.9+."""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
from pathlib import Path
|
|
from queue import Queue, Empty
|
|
import random
|
|
import ssl
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
import uuid
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check, shape
|
|
|
|
|
|
class UploadFailure(Failure):
|
|
def __init__(self, response):
|
|
super().__init__(f'Concurrent upload returned {response.status}, expected 201')
|
|
self.details = {'status': response.status,
|
|
'instance': response.getheader('X-Instance-Id'),
|
|
'requestId': response.getheader('X-Request-Id')}
|
|
|
|
|
|
def docker(*args, **kwargs):
|
|
return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True,
|
|
text=True, check=True, timeout=90, **kwargs).stdout
|
|
|
|
|
|
class MemoryProbe:
|
|
def __init__(self, container):
|
|
self.container = container
|
|
self.messages = Queue()
|
|
source = (ROOT / 'test/upload-memory-probe.mjs').read_text(encoding='utf-8')
|
|
self.process = subprocess.Popen(
|
|
['docker', 'exec', '-i', container, 'node', '--input-type=module', '-e', source],
|
|
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.DEVNULL, text=True, bufsize=1,
|
|
)
|
|
threading.Thread(target=self.read, daemon=True).start()
|
|
|
|
def read(self):
|
|
try:
|
|
for line in self.process.stdout:
|
|
self.messages.put(json.loads(line))
|
|
except (ValueError, OSError):
|
|
self.messages.put({'event': 'error'})
|
|
finally:
|
|
self.messages.put({'event': 'closed'})
|
|
|
|
def receive(self, expected):
|
|
try:
|
|
value = self.messages.get(timeout=45)
|
|
except Empty:
|
|
raise Failure('API memory probe timed out') from None
|
|
check(value.get('event') == expected, 'API memory probe failed: ' + value.get('event', 'unknown'))
|
|
return value
|
|
|
|
def command(self, command, response):
|
|
self.process.stdin.write(command + '\n')
|
|
self.process.stdin.flush()
|
|
return self.receive(response)
|
|
|
|
def close(self):
|
|
if self.process.poll() is None:
|
|
try:
|
|
self.process.stdin.write('quit\n')
|
|
self.process.stdin.flush()
|
|
self.process.stdin.close()
|
|
self.process.wait(timeout=15)
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
self.process.kill()
|
|
self.process.wait(timeout=10)
|
|
self.process.stdout.close()
|
|
|
|
|
|
def chunk(kind, data):
|
|
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
|
|
|
|
|
def png_image(side):
|
|
# Deterministic noise is poorly compressible: the encoded file really occupies RAM.
|
|
rng = random.Random(side)
|
|
rows = b''.join(b'\0' + rng.randbytes(side * 3) for _ in range(side))
|
|
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0))
|
|
+ chunk(b'IDAT', zlib.compress(rows, level=1)))
|
|
|
|
|
|
def upload(base, actor, chat, body, filename, gate, duration, timeout):
|
|
target = urlsplit(base + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename)
|
|
connection_type = http.client.HTTPSConnection if target.scheme == 'https' else http.client.HTTPConnection
|
|
kwargs = {'context': ssl.create_default_context()} if target.scheme == 'https' else {}
|
|
connection = connection_type(target.hostname, target.port, timeout=timeout, **kwargs)
|
|
try:
|
|
connection.putrequest('POST', target.path + '?' + target.query)
|
|
connection.putheader('Authorization', 'Bearer ' + actor['token'])
|
|
connection.putheader('Content-Type', 'image/png')
|
|
connection.putheader('Content-Length', str(len(body)))
|
|
connection.endheaders()
|
|
first = min(65536, len(body) - 1)
|
|
connection.send(body[:first])
|
|
# Every connection has sent headers and an incomplete body before any may finish.
|
|
gate.wait(timeout=timeout)
|
|
started = time.monotonic()
|
|
for offset in range(first, len(body), 65536):
|
|
end = min(offset + 65536, len(body))
|
|
deadline = started + duration * (end - first) / (len(body) - first)
|
|
time.sleep(max(0, deadline - time.monotonic()))
|
|
connection.send(body[offset:end])
|
|
response = connection.getresponse()
|
|
data = response.read(1024 * 1024 + 1)
|
|
if response.status != 201:
|
|
raise UploadFailure(response)
|
|
check(len(data) <= 1024 * 1024, 'Upload response unexpectedly large')
|
|
check(response.getheader('Content-Type', '').startswith('application/json'), 'Upload response is not JSON')
|
|
check(response.getheader('X-Request-Id'), 'Upload lacks request identifier')
|
|
instance = response.getheader('X-Instance-Id')
|
|
check(instance, 'Upload lacks instance identifier')
|
|
attachment = shape(json.loads(data), 'Attachment')
|
|
check(attachment['status'] == 'ready', 'Concurrent attachment is not ready')
|
|
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == actor['id'], 'Incorrect attachment ownership')
|
|
check(attachment['filename'] == filename and attachment['content_type'] == 'image/png', 'Incorrect attachment metadata')
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(body), 'Upload bytes/metadata mismatch')
|
|
return {'attachment': attachment, 'instance': instance, 'elapsedSeconds': time.monotonic() - started,
|
|
'sha256': digest, 'sizeBytes': len(body)}
|
|
except BaseException:
|
|
gate.abort()
|
|
raise
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default='http://localhost:8080')
|
|
parser.add_argument('--concurrency', type=int, default=10)
|
|
parser.add_argument('--rounds', type=int, default=2)
|
|
parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792], metavar='SIDE_PX')
|
|
parser.add_argument('--upload-seconds', type=float, default=2)
|
|
parser.add_argument('--timeout', type=float, default=60)
|
|
parser.add_argument('--max-rss-mib', type=float, help='Optional assertion budget per API process, not a container resource limit')
|
|
parser.add_argument('--output', type=Path, default=ROOT / 'evidence/concurrent-uploads.json')
|
|
args = parser.parse_args()
|
|
check(10 <= args.concurrency <= 64, '--concurrency must be between 10 and 64')
|
|
check(1 <= args.rounds <= 10, '--rounds must be between 1 and 10')
|
|
check(all(64 <= side <= 1800 for side in args.sizes), '--sizes must be between 64 and 1800 pixels')
|
|
check(0.5 <= args.upload_seconds <= 10 and args.timeout > args.upload_seconds + 5, 'Invalid upload timing')
|
|
check(args.max_rss_mib is None or args.max_rss_mib > 0, 'RSS budget must be positive')
|
|
client = Client(args.base_url, timeout=args.timeout)
|
|
client.call('GET', '/health/ready', schema='Health')
|
|
run_id = str(uuid.uuid4())
|
|
report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(),
|
|
'concurrency': args.concurrency, 'roundsPerSize': args.rounds,
|
|
'uploadSeconds': args.upload_seconds, 'maxRssMiB': args.max_rss_mib,
|
|
'memoryMethod': 'API /proc VmRSS sampled every 50 ms; VmHWM is lifetime, not reset; probe RSS excluded',
|
|
'waves': []}
|
|
actors, probes, chat = [], [], None
|
|
try:
|
|
containers = docker('compose', 'ps', '-q', 'api').split()
|
|
check(len(containers) >= 2, 'Expected at least two running API replicas')
|
|
before = {state['Id']: (state['State']['StartedAt'], state['RestartCount'])
|
|
for state in json.loads(docker('inspect', *containers))}
|
|
for container in containers:
|
|
probe = MemoryProbe(container)
|
|
probes.append(probe)
|
|
probe.ready = probe.receive('ready')
|
|
owner = client.user('cu' + run_id[:8] + 'a')
|
|
actors.append(owner)
|
|
reader = client.user('cu' + run_id[:8] + 'b')
|
|
actors.append(reader)
|
|
chat = client.call('POST', '/api/v1/chats', 201, actor=owner,
|
|
body={'title': 'concurrency:' + run_id, 'member_ids': [actor['id'] for actor in actors]},
|
|
schema='Chat')['json']
|
|
for side in args.sizes:
|
|
image = png_image(side)
|
|
for number in range(1, args.rounds + 1):
|
|
bodies = [image + chunk(b'tEXt', f'fixture\0{run_id}-{side}-{number}-{i}'.encode()) + chunk(b'IEND', b'')
|
|
for i in range(args.concurrency)]
|
|
check(all(len(body) <= 10485760 for body in bodies), 'Generated PNG exceeds 10 MiB')
|
|
wave = {'sidePixels': side, 'round': number, 'fileBytes': len(bodies[0]),
|
|
'totalBytes': sum(map(len, bodies)), 'incompleteUploadsAtBarrier': 0,
|
|
'successfulUploads': 0, 'verifiedDownloads': 0, 'memory': []}
|
|
report['waves'].append(wave)
|
|
for probe in probes:
|
|
probe.command('begin', 'begin')
|
|
gate = threading.Barrier(args.concurrency + 1)
|
|
started = time.monotonic()
|
|
futures = []
|
|
print(f'Uploading: {args.concurrency} x {len(bodies[0]) / 1048576:.2f} MiB, round {number}', flush=True)
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
futures = [pool.submit(upload, client.base, owner, chat, body,
|
|
f'{side}-{number}-{i}.png', gate, args.upload_seconds, args.timeout)
|
|
for i, body in enumerate(bodies)]
|
|
gate.wait(timeout=args.timeout)
|
|
wave['incompleteUploadsAtBarrier'] = args.concurrency
|
|
results = [future.result() for future in futures]
|
|
finally:
|
|
wave['uploadWallSeconds'] = time.monotonic() - started
|
|
memory = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes]
|
|
wave['memory'] = memory
|
|
wave['successfulUploads'] = sum(future.done() and not future.cancelled() and future.exception() is None for future in futures)
|
|
wave['failedUploads'] = [
|
|
{'index': index, 'errorType': type(future.exception()).__name__,
|
|
**getattr(future.exception(), 'details', {})}
|
|
for index, future in enumerate(futures)
|
|
if future.done() and not future.cancelled() and future.exception() is not None
|
|
]
|
|
wave['instances'] = {name: sum(r['instance'] == name for r in results) for name in sorted({r['instance'] for r in results})}
|
|
wave['latencySeconds'] = {'min': min(r['elapsedSeconds'] for r in results), 'max': max(r['elapsedSeconds'] for r in results)}
|
|
check(len({r['attachment']['id'] for r in results}) == args.concurrency, 'Attachment IDs collide')
|
|
check(len(wave['instances']) >= 2, 'Concurrent uploads did not reach both replicas')
|
|
check(set(wave['instances']).issubset({probe.ready['instance'] for probe in probes}), 'Unmeasured API replica served uploads')
|
|
for sample in memory:
|
|
check(sample['samples'] >= 2, 'Insufficient memory samples during upload')
|
|
check(sample['oomDelta'] == 0 and sample['oomKillDelta'] == 0, 'API cgroup reported OOM during upload')
|
|
if args.max_rss_mib is not None:
|
|
check(sample['sampledPeakRssBytes'] <= args.max_rss_mib * 1048576, 'API sampled RSS exceeded assertion budget')
|
|
for result in results:
|
|
path = '/api/v1/attachments/' + result['attachment']['id']
|
|
meta = client.call('GET', path, actor=reader, schema='Attachment')['json']
|
|
check(meta == result['attachment'], 'Concurrent metadata changed')
|
|
data = client.call('GET', path + '/content', actor=reader)['bytes']
|
|
check(len(data) == result['sizeBytes'] and hashlib.sha256(data).hexdigest() == result['sha256'], 'Concurrent download checksum mismatch')
|
|
wave['verifiedDownloads'] += 1
|
|
peaks = ', '.join(f"{s['instance']}: {s['sampledPeakRssBytes'] / 1048576:.1f} MiB" for s in memory)
|
|
print('PASS: all uploads/downloads verified; sampled API RSS peaks: ' + peaks, flush=True)
|
|
after = json.loads(docker('inspect', *containers))
|
|
check(all(before[state['Id']] == (state['State']['StartedAt'], state['RestartCount'])
|
|
and state['State']['Running'] and not state['State']['OOMKilled'] for state in after), 'API restarted or was OOM-killed')
|
|
client.call('GET', '/health/ready', schema='Health')
|
|
report['status'] = 'passed'
|
|
except BaseException as error:
|
|
report['status'] = 'failed'
|
|
report['errorType'] = type(error).__name__
|
|
if isinstance(error, Failure):
|
|
report['failure'] = str(error)
|
|
raise
|
|
finally:
|
|
for probe in probes:
|
|
if hasattr(probe, 'process'):
|
|
try:
|
|
probe.close()
|
|
except (OSError, subprocess.SubprocessError):
|
|
report['probeCloseFailed'] = True
|
|
try:
|
|
source = (ROOT / 'test/concurrent-cleanup.mjs').read_text(encoding='utf-8')
|
|
payload = json.dumps({'runId': run_id, 'chatId': chat['id'] if chat else None,
|
|
'userIds': [actor['id'] for actor in actors]})
|
|
docker('compose', 'exec', '-T', '--index', '1', 'api', 'node', '--input-type=module', '-e', source, input=payload)
|
|
report['fixturesRemoved'] = True
|
|
except BaseException:
|
|
report['fixturesRemoved'] = False
|
|
if report['status'] == 'passed':
|
|
report['status'] = 'failed'
|
|
raise Failure('Concurrent fixtures cleanup failed') from None
|
|
finally:
|
|
report['finishedAt'] = datetime.now(timezone.utc).isoformat()
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
|
|
print('PASS: concurrent uploads, byte integrity, memory sampling and no API restarts/OOM', flush=True)
|
|
print('Report: ' + str(args.output))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
try:
|
|
main()
|
|
except (Failure, ValueError, OSError, subprocess.SubprocessError, threading.BrokenBarrierError) as error:
|
|
# Never include subprocess arguments or request headers: they can contain credentials.
|
|
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
|
|
sys.exit(1)
|