Files
mtusi2026lab4/scripts/lab4/upload-memory.py
T

377 lines
19 KiB
Python

#!/usr/bin/env python3
"""Concurrent upload experiment; stdlib only, no stand lifecycle operations."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import random
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from urllib.parse import urlsplit
import zlib
ROOT = Path(__file__).resolve().parents[2]
sys.dont_write_bytecode = True
CASES = (
('small', 512, 512, 1024 * 1024, 41),
('limit', 1600, 1600, 10 * 1024 * 1024, 42),
)
MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None)
def png_chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def make_png(width, height, size, seed):
rng = random.Random(seed)
rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1))
ending = png_chunk(b'IEND', b'')
gap = size - len(prefix) - len(ending)
if gap < 0 or 0 < gap < 12:
raise ValueError('target cannot contain this PNG')
# Ancillary, private, reserved bit valid: unknown npAD is safely ignored.
padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b''
return prefix + padding + ending
def make_rgba64_png(width, height):
encoder = zlib.compressobj(1)
row = bytes(1 + width * 8)
pieces = [encoder.compress(row) for _ in range(height)]
pieces.append(encoder.flush())
header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header)
+ png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b''))
def payload_for(case):
name, width, height, size, seed = case
return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed)
def memory_bytes(value):
match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip())
if not match:
raise ValueError('unrecognized docker memory value')
units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2,
'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4}
return round(float(match[1]) * units[match[2]])
def summarize_memory(samples, container_ids):
result = {}
for container_id in container_ids:
baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline']
load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load']
if not baseline or not load:
raise RuntimeError('missing baseline or load sample for an API replica')
before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load)
result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before,
'baseline_samples': len(baseline), 'load_samples': len(load)}
return result
def latency_summary(values):
ordered = sorted(values)
return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1],
'max_ms': ordered[-1]}
def max_inflight(intervals):
active, peak = 0, 0
events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals]
for _, delta in sorted(events):
active += delta
peak = max(peak, active)
return peak
class DockerStats:
def __init__(self, container_ids):
self.ids = container_ids
self.samples = []
self.lock = threading.Condition()
self.case, self.phase = '', 'idle'
self.error = None
self.started = time.monotonic()
def __enter__(self):
self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, encoding='utf-8', errors='replace', bufsize=1)
self.thread = threading.Thread(target=self._read, daemon=True)
self.thread.start()
return self
def _read(self):
for line in self.process.stdout:
try:
cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
if not cleaned:
continue
raw = json.loads(cleaned)
reported_id = raw.get('ID') or raw.get('Container', '')
container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid))
used = memory_bytes(raw['MemUsage'])
limit = memory_bytes(raw['MemUsage'].split('/')[1])
if used == 0:
continue
with self.lock:
self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used,
'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6),
'case': self.case, 'phase': self.phase})
self.lock.notify_all()
except (ValueError, KeyError, IndexError, StopIteration) as exc:
with self.lock:
self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120])
self.lock.notify_all()
with self.lock:
self.lock.notify_all()
def mark(self, case, phase):
with self.lock:
self.case, self.phase = case, phase
def snapshot(self, case):
with self.lock:
return [dict(s) for s in self.samples if s['case'] == case]
def wait_for_baseline(self, case, timeout=20):
deadline = time.monotonic() + timeout
with self.lock:
while True:
seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'}
if seen == set(self.ids):
return
if self.error or self.process.poll() is not None or time.monotonic() >= deadline:
raise RuntimeError('docker stats baseline unavailable')
self.lock.wait(min(.1, max(0, deadline - time.monotonic())))
def __exit__(self, *_):
if self.process.poll() is None:
self.process.terminate()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
self.process.kill()
self.process.wait(timeout=5)
self.thread.join(timeout=5)
self.process.stdout.close()
def api_containers(args):
command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)]
if args.project_name:
command.extend(['--project-name', args.project_name])
result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
ids = result.stdout.split()
if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids):
raise RuntimeError('compose has no running API containers')
return ids
def container_states(container_ids):
template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},'
'"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},'
'"exit_code":{{json .State.ExitCode}}}')
result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())}
def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'):
result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)}
barrier.wait(timeout=30)
started = time.monotonic()
try:
suffix = '.jpg' if mime == 'image/jpeg' else '.png'
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix
req = urllib.request.Request(client.base + path, data=content, method='POST', headers={
'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime,
'Content-Length': str(len(content)), 'Accept': 'application/json'})
try:
response = client.opener.open(req, timeout=client.timeout)
except urllib.error.HTTPError as exc:
response = exc
with response:
result['status'] = response.code
result['instance_id'] = response.headers.get('X-Instance-Id')
body = response.read(65537)
if result['status'] != 201:
result['error'] = 'http_error'
elif len(body) > 65536:
result['error'] = 'oversized_metadata'
else:
attachment = json.loads(body)
client_module.shape(attachment, 'Attachment')
valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content)
and attachment['status'] == 'ready' and attachment['owner_id'] == actor['id']
and attachment['chat_id'] == chat['id'])
result['ok'] = valid
result['attachment_id'] = attachment['id']
if not valid:
result['error'] = 'attachment_metadata_mismatch'
except Exception as exc:
# Exception text and response bodies are deliberately excluded from evidence.
result['error'] = type(exc).__name__
finished = time.monotonic()
result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3))
return result
def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case):
name, width, height, size, seed = case
content = payload_for(case)
size = len(content)
checksum = hashlib.sha256(content).hexdigest()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
state_before = container_states(stats.ids)
stats.mark(name, 'baseline')
stats.wait_for_baseline(name)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name)
for index, client in enumerate(clients)]
stats.mark(name, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
duration = time.monotonic() - started
# Retain a tail because docker stats reports its own sampled windows.
time.sleep(args.tail_seconds)
stats.mark(name, 'idle')
samples = stats.snapshot(name)
memory_error = None
try:
memory = summarize_memory(samples, stats.ids)
except RuntimeError as exc:
memory, memory_error = {}, str(exc)
intervals = [(r['_started'], r['_finished']) for r in results]
peak_requests = max_inflight(intervals)
successes = [r for r in results if r['ok']]
download = {'ok': False}
if successes:
try:
response = bootstrap_client.call('GET', '/api/v1/attachments/' + successes[0]['attachment_id'] + '/content', actor=reader)
downloaded = response['bytes']
download = {'ok': downloaded == content and response['headers'].get_content_type() == 'image/png',
'bytes': len(downloaded), 'sha256': hashlib.sha256(downloaded).hexdigest(),
'status': response['status']}
except Exception as exc:
download['error'] = type(exc).__name__
distinct = len({r['attachment_id'] for r in successes})
state_after = container_states(stats.ids)
stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed']
and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum,
'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2,
'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4),
'requests': args.concurrency, 'attempted_bytes': size * args.concurrency,
'successful_bytes': size * len(successes), 'accepted': len(successes),
'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)),
'errors': len(results) - len(successes), 'duration_s': round(duration, 6),
'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct,
'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory,
'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download,
'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable,
'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10
and download['ok'] and not memory_error and stable)}
def main():
parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml')
parser.add_argument('--project-name', default=None)
parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here')
parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard',
help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)')
parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker')
args = parser.parse_args()
selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,)
if args.check_generator:
print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload),
'sha256': hashlib.sha256(payload).hexdigest()}
for name, width, height, size, seed in selected_cases
for payload in [payload_for((name, width, height, size, seed))]], indent=2))
return 0
if not args.output:
parser.error('--output is required for live execution')
if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s')
target = urlsplit(args.base_url)
if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment:
parser.error('--base-url must be HTTP(S) without credentials, query or fragment')
report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False,
'driver': {'os': platform.system(), 'architecture': platform.machine(),
'logical_cpus': os.cpu_count(), 'python': platform.python_version()},
'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second',
'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail',
'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds,
'same_bytes_per_case': True, 'p95_method': 'nearest rank',
'profile': args.profile,
'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard'
else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []}
try:
sys.path.insert(0, str(ROOT / 'tests'))
import client as client_module
ids = api_containers(args)
report['api_container_ids'] = ids
bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout)
report['stage'] = 'bootstrap'
actor, reader, _, chat = bootstrap_client.bootstrap()
with DockerStats(ids) as stats:
for case in selected_cases:
report['stage'] = case[0]
report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case))
if stats.error:
report['sampling_error'] = stats.error
report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error
report['stage'] = 'complete'
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)):
# Public helper diagnostics contain only paths/schema names, never values or credentials.
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'output': str(args.output),
'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'],
'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'],
'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']],
'error': report.get('error')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())