#!/usr/bin/env python3 """File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits. Run with python -X utf8 -B. Default is one upload; no container lifecycle actions. """ import argparse from collections import Counter from concurrent.futures import ThreadPoolExecutor import datetime as dt import hashlib import importlib.util import json from pathlib import Path import re import struct import subprocess import sys import threading import time from urllib.parse import urlsplit import zlib sys.dont_write_bytecode = True _spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) memory = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(memory) def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536): """Stream zero scanlines, including filter bytes; never allocate a full row.""" if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000: raise ValueError('invalid stress dimensions or chunk size') path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) encoder = zlib.compressobj(1) remaining = height * (width * 8 + 1) zeros = bytes(min(chunk_bytes, remaining)) with path.open('wb') as out: out.write(b'\x89PNG\r\n\x1a\n') out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0))) while remaining: count = min(len(zeros), remaining) compressed = encoder.compress(zeros[:count]) if compressed: out.write(memory.png_chunk(b'IDAT', compressed)) remaining -= count out.write(memory.png_chunk(b'IDAT', encoder.flush())) out.write(memory.png_chunk(b'IEND', b'')) def fixture_info(content, mime): if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33: width, height, depth, color = struct.unpack_from('>IIBB', content, 16) return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color} if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'): pos = 2 while pos + 4 <= len(content): if content[pos] != 255: break while pos < len(content) and content[pos] == 255: pos += 1 marker = content[pos] pos += 1 length = struct.unpack_from('>H', content, pos)[0] frame = content[pos + 2:pos + length] pos += length if marker in (0xc0, 0xc1, 0xc2): height, width = struct.unpack_from('>HH', frame, 1) count = frame[5] return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count, 'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]} if length < 2 or marker == 0xda: break raise ValueError('fixture MIME/header mismatch or missing image header') def read_fixture(path, mime): if not 0 < path.stat().st_size <= 10 * 1024 * 1024: raise ValueError('fixture must be between 1 byte and 10 MiB') content = path.read_bytes() info = fixture_info(content, mime) if not 0 < info['width'] * info['height'] <= 40_000_000: raise ValueError('fixture exceeds 40M pixels') return content, info def api_limits(ids): template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}' result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())} def experiment(args, content, report): sys.path.insert(0, str(memory.ROOT / 'tests')) import client as module ids = memory.api_containers(args) report['api_limits'] = api_limits(ids) if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids): raise RuntimeError('stress uploads require every API container limit to be >=2 GiB') report['state_before'] = memory.container_states(ids) bootstrap = module.Client(args.base_url, timeout=args.timeout) actor, reader, _, chat = bootstrap.bootstrap() clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] with memory.DockerStats(ids) as stats: stats.mark(args.label, 'baseline') stats.wait_for_baseline(args.label) time.sleep(args.baseline_seconds) barrier = threading.Barrier(args.concurrency + 1) with ThreadPoolExecutor(max_workers=args.concurrency) as executor: futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'], barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)] stats.mark(args.label, 'load') started = time.monotonic() barrier.wait(timeout=30) results = [future.result() for future in futures] report['duration_s'] = time.monotonic() - started time.sleep(args.tail_seconds) stats.mark(args.label, 'idle') report['memory_samples'] = stats.snapshot(args.label) report['sampling_error'] = stats.error report['state_after'] = memory.container_states(ids) stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed'] and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids) report['restart_or_oom_detected'] = not stable try: report['memory'] = memory.summarize_memory(report['memory_samples'], ids) except RuntimeError as exc: report['memory_error'] = str(exc) report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results]) accepted = [r for r in results if r['ok']] report['accepted'] = len(accepted) report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)) report['latency'] = memory.latency_summary([r['latency_ms'] for r in results]) report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted) for result in results: result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) result.pop('_finished') report['results'] = results report['download'] = {'ok': False} if accepted: try: response = bootstrap.call('GET', '/api/v1/attachments/' + accepted[0]['attachment_id'] + '/content', actor=reader) body = response['bytes'] report['download'] = {'ok': body == content and response['headers'].get_content_type() == args.mime, 'status': response['status'], 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest()} except Exception as exc: report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__ report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable and not report.get('memory_error') and not report['sampling_error'] and report['observed_max_inflight'] == args.concurrency) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--file', type=Path) parser.add_argument('--mime', choices=('image/png', 'image/jpeg')) parser.add_argument('--label', default='decoder') parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance') parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml') parser.add_argument('--project-name') parser.add_argument('--timeout', type=float, default=45) parser.add_argument('--baseline-seconds', type=float, default=3) parser.add_argument('--tail-seconds', type=float, default=3) parser.add_argument('--output', type=Path) parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker') parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input') args = parser.parse_args() if args.generate_wide: write_rgba64_png(args.generate_wide) content, info = read_fixture(args.generate_wide, 'image/png') print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})) return 0 if not args.file or not args.mime: parser.error('--file and --mime are required') if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s') if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label): parser.error('label must be 1-40 safe ASCII characters') url = urlsplit(args.base_url) if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment: parser.error('base URL must be HTTP(S) without credentials/query/fragment') content, info = read_fixture(args.file, args.mime) report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info, 'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(), 'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'measurement': {'source': 'docker stats JSON stream, formatted working-set values', 'limitation': 'sampled/rounded peaks can miss instantaneous maxima', 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}} if args.check_file: print(json.dumps(report, indent=2)) return 0 if not args.output: parser.error('--output is required for a live run') try: experiment(args, content, report) except Exception as exc: report['error'] = type(exc).__name__ if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure': report['failure'] = str(exc) args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')})) return 0 if report['ok'] else 1 if __name__ == '__main__': raise SystemExit(main())