"""Concurrent raw PNG uploads through Nginx with per-API RSS measurements. Python 3.9+.""" import argparse from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone import hashlib import http.client import json from pathlib import Path from queue import Queue, Empty import random import ssl import struct import subprocess import sys import threading import time from urllib.parse import urlsplit import uuid import zlib ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check, shape class UploadFailure(Failure): def __init__(self, response): super().__init__(f'Concurrent upload returned {response.status}, expected 201') self.details = {'status': response.status, 'instance': response.getheader('X-Instance-Id'), 'requestId': response.getheader('X-Request-Id')} def docker(*args, **kwargs): return subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, check=True, timeout=90, **kwargs).stdout class MemoryProbe: def __init__(self, container): self.container = container self.messages = Queue() source = (ROOT / 'test/upload-memory-probe.mjs').read_text(encoding='utf-8') self.process = subprocess.Popen( ['docker', 'exec', '-i', container, 'node', '--input-type=module', '-e', source], cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1, ) threading.Thread(target=self.read, daemon=True).start() def read(self): try: for line in self.process.stdout: self.messages.put(json.loads(line)) except (ValueError, OSError): self.messages.put({'event': 'error'}) finally: self.messages.put({'event': 'closed'}) def receive(self, expected): try: value = self.messages.get(timeout=45) except Empty: raise Failure('API memory probe timed out') from None check(value.get('event') == expected, 'API memory probe failed: ' + value.get('event', 'unknown')) return value def command(self, command, response): self.process.stdin.write(command + '\n') self.process.stdin.flush() return self.receive(response) def close(self): if self.process.poll() is None: try: self.process.stdin.write('quit\n') self.process.stdin.flush() self.process.stdin.close() self.process.wait(timeout=15) except (OSError, subprocess.TimeoutExpired): self.process.kill() self.process.wait(timeout=10) self.process.stdout.close() def chunk(kind, data): return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) def png_image(side): # Deterministic noise is poorly compressible: the encoded file really occupies RAM. rng = random.Random(side) rows = b''.join(b'\0' + rng.randbytes(side * 3) for _ in range(side)) return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(rows, level=1))) def upload(base, actor, chat, body, filename, gate, duration, timeout): target = urlsplit(base + '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename) connection_type = http.client.HTTPSConnection if target.scheme == 'https' else http.client.HTTPConnection kwargs = {'context': ssl.create_default_context()} if target.scheme == 'https' else {} connection = connection_type(target.hostname, target.port, timeout=timeout, **kwargs) try: connection.putrequest('POST', target.path + '?' + target.query) connection.putheader('Authorization', 'Bearer ' + actor['token']) connection.putheader('Content-Type', 'image/png') connection.putheader('Content-Length', str(len(body))) connection.endheaders() first = min(65536, len(body) - 1) connection.send(body[:first]) # Every connection has sent headers and an incomplete body before any may finish. gate.wait(timeout=timeout) started = time.monotonic() for offset in range(first, len(body), 65536): end = min(offset + 65536, len(body)) deadline = started + duration * (end - first) / (len(body) - first) time.sleep(max(0, deadline - time.monotonic())) connection.send(body[offset:end]) response = connection.getresponse() data = response.read(1024 * 1024 + 1) if response.status != 201: raise UploadFailure(response) check(len(data) <= 1024 * 1024, 'Upload response unexpectedly large') check(response.getheader('Content-Type', '').startswith('application/json'), 'Upload response is not JSON') check(response.getheader('X-Request-Id'), 'Upload lacks request identifier') instance = response.getheader('X-Instance-Id') check(instance, 'Upload lacks instance identifier') attachment = shape(json.loads(data), 'Attachment') check(attachment['status'] == 'ready', 'Concurrent attachment is not ready') check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == actor['id'], 'Incorrect attachment ownership') check(attachment['filename'] == filename and attachment['content_type'] == 'image/png', 'Incorrect attachment metadata') digest = hashlib.sha256(body).hexdigest() check(attachment['sha256'] == digest and attachment['size_bytes'] == len(body), 'Upload bytes/metadata mismatch') return {'attachment': attachment, 'instance': instance, 'elapsedSeconds': time.monotonic() - started, 'sha256': digest, 'sizeBytes': len(body)} except BaseException: gate.abort() raise finally: connection.close() def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--concurrency', type=int, default=10) parser.add_argument('--rounds', type=int, default=2) parser.add_argument('--sizes', type=int, nargs='+', default=[512, 1792], metavar='SIDE_PX') parser.add_argument('--upload-seconds', type=float, default=2) parser.add_argument('--timeout', type=float, default=60) parser.add_argument('--max-rss-mib', type=float, help='Optional assertion budget per API process, not a container resource limit') parser.add_argument('--output', type=Path, default=ROOT / 'evidence/concurrent-uploads.json') args = parser.parse_args() check(10 <= args.concurrency <= 64, '--concurrency must be between 10 and 64') check(1 <= args.rounds <= 10, '--rounds must be between 1 and 10') check(all(64 <= side <= 1800 for side in args.sizes), '--sizes must be between 64 and 1800 pixels') check(0.5 <= args.upload_seconds <= 10 and args.timeout > args.upload_seconds + 5, 'Invalid upload timing') check(args.max_rss_mib is None or args.max_rss_mib > 0, 'RSS budget must be positive') client = Client(args.base_url, timeout=args.timeout) client.call('GET', '/health/ready', schema='Health') run_id = str(uuid.uuid4()) report = {'status': 'running', 'startedAt': datetime.now(timezone.utc).isoformat(), 'concurrency': args.concurrency, 'roundsPerSize': args.rounds, 'uploadSeconds': args.upload_seconds, 'maxRssMiB': args.max_rss_mib, 'memoryMethod': 'API /proc VmRSS sampled every 50 ms; VmHWM is lifetime, not reset; probe RSS excluded', 'waves': []} actors, probes, chat = [], [], None try: containers = docker('compose', 'ps', '-q', 'api').split() check(len(containers) >= 2, 'Expected at least two running API replicas') before = {state['Id']: (state['State']['StartedAt'], state['RestartCount']) for state in json.loads(docker('inspect', *containers))} for container in containers: probe = MemoryProbe(container) probes.append(probe) probe.ready = probe.receive('ready') owner = client.user('cu' + run_id[:8] + 'a') actors.append(owner) reader = client.user('cu' + run_id[:8] + 'b') actors.append(reader) chat = client.call('POST', '/api/v1/chats', 201, actor=owner, body={'title': 'concurrency:' + run_id, 'member_ids': [actor['id'] for actor in actors]}, schema='Chat')['json'] for side in args.sizes: image = png_image(side) for number in range(1, args.rounds + 1): bodies = [image + chunk(b'tEXt', f'fixture\0{run_id}-{side}-{number}-{i}'.encode()) + chunk(b'IEND', b'') for i in range(args.concurrency)] check(all(len(body) <= 10485760 for body in bodies), 'Generated PNG exceeds 10 MiB') wave = {'sidePixels': side, 'round': number, 'fileBytes': len(bodies[0]), 'totalBytes': sum(map(len, bodies)), 'incompleteUploadsAtBarrier': 0, 'successfulUploads': 0, 'verifiedDownloads': 0, 'memory': []} report['waves'].append(wave) for probe in probes: probe.command('begin', 'begin') gate = threading.Barrier(args.concurrency + 1) started = time.monotonic() futures = [] print(f'Uploading: {args.concurrency} x {len(bodies[0]) / 1048576:.2f} MiB, round {number}', flush=True) try: with ThreadPoolExecutor(max_workers=args.concurrency) as pool: futures = [pool.submit(upload, client.base, owner, chat, body, f'{side}-{number}-{i}.png', gate, args.upload_seconds, args.timeout) for i, body in enumerate(bodies)] gate.wait(timeout=args.timeout) wave['incompleteUploadsAtBarrier'] = args.concurrency results = [future.result() for future in futures] finally: wave['uploadWallSeconds'] = time.monotonic() - started memory = [{'instance': probe.ready['instance'], **probe.command('end', 'summary')} for probe in probes] wave['memory'] = memory wave['successfulUploads'] = sum(future.done() and not future.cancelled() and future.exception() is None for future in futures) wave['failedUploads'] = [ {'index': index, 'errorType': type(future.exception()).__name__, **getattr(future.exception(), 'details', {})} for index, future in enumerate(futures) if future.done() and not future.cancelled() and future.exception() is not None ] wave['instances'] = {name: sum(r['instance'] == name for r in results) for name in sorted({r['instance'] for r in results})} wave['latencySeconds'] = {'min': min(r['elapsedSeconds'] for r in results), 'max': max(r['elapsedSeconds'] for r in results)} check(len({r['attachment']['id'] for r in results}) == args.concurrency, 'Attachment IDs collide') check(len(wave['instances']) >= 2, 'Concurrent uploads did not reach both replicas') check(set(wave['instances']).issubset({probe.ready['instance'] for probe in probes}), 'Unmeasured API replica served uploads') for sample in memory: check(sample['samples'] >= 2, 'Insufficient memory samples during upload') check(sample['oomDelta'] == 0 and sample['oomKillDelta'] == 0, 'API cgroup reported OOM during upload') if args.max_rss_mib is not None: check(sample['sampledPeakRssBytes'] <= args.max_rss_mib * 1048576, 'API sampled RSS exceeded assertion budget') for result in results: path = '/api/v1/attachments/' + result['attachment']['id'] meta = client.call('GET', path, actor=reader, schema='Attachment')['json'] check(meta == result['attachment'], 'Concurrent metadata changed') data = client.call('GET', path + '/content', actor=reader)['bytes'] check(len(data) == result['sizeBytes'] and hashlib.sha256(data).hexdigest() == result['sha256'], 'Concurrent download checksum mismatch') wave['verifiedDownloads'] += 1 peaks = ', '.join(f"{s['instance']}: {s['sampledPeakRssBytes'] / 1048576:.1f} MiB" for s in memory) print('PASS: all uploads/downloads verified; sampled API RSS peaks: ' + peaks, flush=True) after = json.loads(docker('inspect', *containers)) check(all(before[state['Id']] == (state['State']['StartedAt'], state['RestartCount']) and state['State']['Running'] and not state['State']['OOMKilled'] for state in after), 'API restarted or was OOM-killed') client.call('GET', '/health/ready', schema='Health') report['status'] = 'passed' except BaseException as error: report['status'] = 'failed' report['errorType'] = type(error).__name__ if isinstance(error, Failure): report['failure'] = str(error) raise finally: for probe in probes: if hasattr(probe, 'process'): try: probe.close() except (OSError, subprocess.SubprocessError): report['probeCloseFailed'] = True try: source = (ROOT / 'test/concurrent-cleanup.mjs').read_text(encoding='utf-8') payload = json.dumps({'runId': run_id, 'chatId': chat['id'] if chat else None, 'userIds': [actor['id'] for actor in actors]}) docker('compose', 'exec', '-T', '--index', '1', 'api', 'node', '--input-type=module', '-e', source, input=payload) report['fixturesRemoved'] = True except BaseException: report['fixturesRemoved'] = False if report['status'] == 'passed': report['status'] = 'failed' raise Failure('Concurrent fixtures cleanup failed') from None finally: report['finishedAt'] = datetime.now(timezone.utc).isoformat() args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') print('PASS: concurrent uploads, byte integrity, memory sampling and no API restarts/OOM', flush=True) print('Report: ' + str(args.output)) if __name__ == '__main__': try: main() except (Failure, ValueError, OSError, subprocess.SubprocessError, threading.BrokenBarrierError) as error: # Never include subprocess arguments or request headers: they can contain credentials. print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr) sys.exit(1)