Files
mtusi2026lab5/scripts/lab4/decoder-memory.py
T

207 lines
10 KiB
Python

#!/usr/bin/env python3
"""File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits.
Run with python -X utf8 -B. Default is one upload; no container lifecycle actions.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import importlib.util
import json
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import zlib
sys.dont_write_bytecode = True
_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py'))
memory = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(memory)
def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536):
"""Stream zero scanlines, including filter bytes; never allocate a full row."""
if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000:
raise ValueError('invalid stress dimensions or chunk size')
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
encoder = zlib.compressobj(1)
remaining = height * (width * 8 + 1)
zeros = bytes(min(chunk_bytes, remaining))
with path.open('wb') as out:
out.write(b'\x89PNG\r\n\x1a\n')
out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)))
while remaining:
count = min(len(zeros), remaining)
compressed = encoder.compress(zeros[:count])
if compressed:
out.write(memory.png_chunk(b'IDAT', compressed))
remaining -= count
out.write(memory.png_chunk(b'IDAT', encoder.flush()))
out.write(memory.png_chunk(b'IEND', b''))
def fixture_info(content, mime):
if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33:
width, height, depth, color = struct.unpack_from('>IIBB', content, 16)
return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color}
if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'):
pos = 2
while pos + 4 <= len(content):
if content[pos] != 255:
break
while pos < len(content) and content[pos] == 255:
pos += 1
marker = content[pos]
pos += 1
length = struct.unpack_from('>H', content, pos)[0]
frame = content[pos + 2:pos + length]
pos += length
if marker in (0xc0, 0xc1, 0xc2):
height, width = struct.unpack_from('>HH', frame, 1)
count = frame[5]
return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count,
'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]}
if length < 2 or marker == 0xda:
break
raise ValueError('fixture MIME/header mismatch or missing image header')
def read_fixture(path, mime):
if not 0 < path.stat().st_size <= 10 * 1024 * 1024:
raise ValueError('fixture must be between 1 byte and 10 MiB')
content = path.read_bytes()
info = fixture_info(content, mime)
if not 0 < info['width'] * info['height'] <= 40_000_000:
raise ValueError('fixture exceeds 40M pixels')
return content, info
def api_limits(ids):
template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}'
result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())}
def experiment(args, content, report):
sys.path.insert(0, str(memory.ROOT / 'tests'))
import client as module
ids = memory.api_containers(args)
report['api_limits'] = api_limits(ids)
if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids):
raise RuntimeError('stress uploads require every API container limit to be >=2 GiB')
report['state_before'] = memory.container_states(ids)
bootstrap = module.Client(args.base_url, timeout=args.timeout)
actor, reader, _, chat = bootstrap.bootstrap()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
with memory.DockerStats(ids) as stats:
stats.mark(args.label, 'baseline')
stats.wait_for_baseline(args.label)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'],
barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)]
stats.mark(args.label, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
report['duration_s'] = time.monotonic() - started
time.sleep(args.tail_seconds)
stats.mark(args.label, 'idle')
report['memory_samples'] = stats.snapshot(args.label)
report['sampling_error'] = stats.error
report['state_after'] = memory.container_states(ids)
stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed']
and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids)
report['restart_or_oom_detected'] = not stable
try:
report['memory'] = memory.summarize_memory(report['memory_samples'], ids)
except RuntimeError as exc:
report['memory_error'] = str(exc)
report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results])
accepted = [r for r in results if r['ok']]
report['accepted'] = len(accepted)
report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results))
report['latency'] = memory.latency_summary([r['latency_ms'] for r in results])
report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
report['results'] = results
report['download'] = {'ok': False}
if accepted:
try:
report['download'] = memory.verify_download(bootstrap, reader, accepted[0]['attachment_id'], content, args.mime)
except Exception as exc:
report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__
report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable
and not report.get('memory_error') and not report['sampling_error']
and report['observed_max_inflight'] == args.concurrency)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--file', type=Path)
parser.add_argument('--mime', choices=('image/png', 'image/jpeg'))
parser.add_argument('--label', default='decoder')
parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml')
parser.add_argument('--project-name')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--output', type=Path)
parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker')
parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input')
args = parser.parse_args()
if args.generate_wide:
write_rgba64_png(args.generate_wide)
content, info = read_fixture(args.generate_wide, 'image/png')
print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}))
return 0
if not args.file or not args.mime:
parser.error('--file and --mime are required')
if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label):
parser.error('label must be 1-40 safe ASCII characters')
url = urlsplit(args.base_url)
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment:
parser.error('base URL must be HTTP(S) without credentials/query/fragment')
content, info = read_fixture(args.file, args.mime)
report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info,
'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(),
'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'measurement': {'source': 'docker stats JSON stream, formatted working-set values',
'limitation': 'sampled/rounded peaks can miss instantaneous maxima',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}}
if args.check_file:
print(json.dumps(report, indent=2))
return 0
if not args.output:
parser.error('--output is required for a live run')
try:
experiment(args, content, report)
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure':
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())