forked from mtusi2026/mtusi2026lab4
215 lines
10 KiB
Python
215 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
"""Ten overlapping PNG uploads with kernel-recorded memory peaks; stdlib only.
|
|
|
|
Recreates API containers (never volumes) to reset memory high-water marks.
|
|
Run from any directory: python3 scripts/capture_uploads.py.
|
|
"""
|
|
import concurrent.futures
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import platform
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.parse
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
import cluster
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check # noqa: E402
|
|
|
|
WORKERS = 10
|
|
WIDTH, HEIGHT = 3000, 1000
|
|
EVIDENCE = ROOT / 'evidence/concurrent-uploads.json'
|
|
|
|
|
|
def make_png():
|
|
"""Stable, incompressible RGB rows without a host image library."""
|
|
def chunk(kind, data):
|
|
return (struct.pack('!I', len(data)) + kind + data
|
|
+ struct.pack('!I', zlib.crc32(kind + data) & 0xffffffff))
|
|
|
|
compressor = zlib.compressobj(level=0)
|
|
compressed = []
|
|
for row in range(HEIGHT):
|
|
pixels = hashlib.shake_256(b'lab4-concurrent-upload-v1:' + str(row).encode()).digest(WIDTH * 3)
|
|
compressed.append(compressor.compress(b'\x00' + pixels))
|
|
compressed.append(compressor.flush())
|
|
return (b'\x89PNG\r\n\x1a\n'
|
|
+ chunk(b'IHDR', struct.pack('!IIBBBBB', WIDTH, HEIGHT, 8, 2, 0, 0, 0))
|
|
+ chunk(b'IDAT', b''.join(compressed)) + chunk(b'IEND', b''))
|
|
|
|
|
|
def memory(container):
|
|
# cgroup peaks include allocations between observations; VmHWM is PID 1's
|
|
# resident peak, whereas cgroup memory also includes file cache and children.
|
|
code = '''import json,pathlib
|
|
def number(path):
|
|
return int(pathlib.Path(path).read_text().strip())
|
|
root=pathlib.Path('/sys/fs/cgroup')
|
|
if (root/'memory.peak').exists():
|
|
source='cgroup-v2'
|
|
peak=number(root/'memory.peak'); current=number(root/'memory.current')
|
|
events=dict((k,int(v)) for k,v in (line.split() for line in (root/'memory.events').read_text().splitlines()))
|
|
else:
|
|
root=root/'memory'; source='cgroup-v1'
|
|
peak=number(root/'memory.max_usage_in_bytes'); current=number(root/'memory.usage_in_bytes')
|
|
events={'failcnt':number(root/'memory.failcnt')}
|
|
status=dict(line.split(':',1) for line in pathlib.Path('/proc/1/status').read_text().splitlines() if ':' in line)
|
|
print(json.dumps({'source':source,'cgroup_peak_bytes':peak,'cgroup_current_bytes':current,
|
|
'pid1_vmhwm_bytes':int(status['VmHWM'].split()[0])*1024,
|
|
'pid1_vmrss_bytes':int(status['VmRSS'].split()[0])*1024,'memory_events':events}))
|
|
'''
|
|
return json.loads(cluster.command(['docker', 'exec', container, 'python', '-c', code]))
|
|
|
|
|
|
def version_context():
|
|
revision = cluster.command(['git', 'rev-parse', 'HEAD'])
|
|
dirty = subprocess.run(['git', 'diff', '--quiet'], cwd=ROOT, check=False).returncode != 0
|
|
address = urllib.parse.urlsplit(cluster.BASE_URL)
|
|
return {
|
|
'git_commit': revision, 'tracked_worktree_dirty': dirty,
|
|
'host_python': platform.python_version(), 'host_os': platform.platform(),
|
|
'docker_version': cluster.command(['docker', 'version', '--format', '{{.Server.Version}}']),
|
|
'compose_version': cluster.compose('version', '--short'),
|
|
'contract_version': json.loads((ROOT / 'lab.json').read_text())['contract_version'],
|
|
'endpoint': {'scheme': address.scheme, 'host': address.hostname, 'port': address.port},
|
|
}
|
|
|
|
|
|
def experiment(evidence):
|
|
payload = make_png()
|
|
check(9_000_000 <= len(payload) <= 10 * 1024 * 1024, 'generated PNG size outside experiment bounds')
|
|
digest = hashlib.sha256(payload).hexdigest()
|
|
evidence.update({
|
|
'context': version_context(),
|
|
'input': {'format': 'PNG', 'mode': 'RGB', 'width': WIDTH, 'height': HEIGHT,
|
|
'size_bytes': len(payload), 'sha256': digest,
|
|
'generator': 'SHAKE-256 lab4-concurrent-upload-v1 row seeds; filter 0; zlib level 0',
|
|
'same_immutable_bytes_for_all_requests': True},
|
|
'concurrency': WORKERS,
|
|
'reset': {'command': 'docker compose up -d --no-deps --force-recreate api',
|
|
'purpose': 'Reset API cgroup memory.peak and PID 1 VmHWM; retain all volumes'},
|
|
'peak_scope': 'Fresh API startup, bootstrap, ten uploads and representative download; no peak sampling gaps',
|
|
})
|
|
cluster.compose('up', '-d', '--no-deps', '--force-recreate', 'api')
|
|
containers = cluster.container_ids('api')
|
|
check(len(containers) == 2, 'experiment requires exactly two API containers')
|
|
for container in containers:
|
|
cluster.wait_healthy(container)
|
|
cluster.wait_cluster(2)
|
|
replicas = cluster.replicas()
|
|
evidence['cluster_before'] = cluster.snapshot()
|
|
evidence['baseline'] = {item['instance']: memory(item['container']) for item in replicas}
|
|
limits = [item for item in evidence['cluster_before'] if item['service'] == 'api']
|
|
evidence['memory_limits'] = [
|
|
{key: item[key] for key in ('instance', 'memory_bytes', 'nano_cpus')} for item in limits]
|
|
check(all(item['memory_bytes'] == 512 * 1024 * 1024 and item['nano_cpus'] == 1_000_000_000
|
|
for item in limits), 'API limits must be 512 MiB and 1 CPU each')
|
|
client = Client(timeout=180)
|
|
alice, bob, _eve, chat = client.bootstrap()
|
|
barrier = threading.Barrier(WORKERS, timeout=30)
|
|
lock = threading.Lock()
|
|
activity = {'active': 0, 'peak': 0}
|
|
origin = time.monotonic()
|
|
|
|
def upload(index):
|
|
worker = Client(timeout=180)
|
|
barrier.wait()
|
|
with lock:
|
|
start = time.monotonic() - origin
|
|
activity['active'] += 1
|
|
activity['peak'] = max(activity['peak'], activity['active'])
|
|
item = {'index': index, 'start_seconds': start}
|
|
attachment = None
|
|
try:
|
|
response = worker.call(
|
|
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=concurrent-%02d.png' % index,
|
|
expected=201, actor=alice, raw=payload,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
attachment = response['json']
|
|
item.update({'status': response['status'], 'instance': response['headers'].get('X-Instance-Id')})
|
|
check(attachment['sha256'] == digest and attachment['size_bytes'] == len(payload),
|
|
'uploaded original checksum/size mismatch')
|
|
check(attachment['chat_id'] == chat['id'] and attachment['owner_id'] == alice['id'],
|
|
'uploaded attachment ownership mismatch')
|
|
check(attachment['status'] == 'ready', 'uploaded attachment not ready')
|
|
check(attachment['content_type'] == 'image/png', 'uploaded MIME type mismatch')
|
|
item.update({'attachment_id': attachment['id'], 'metadata_verified': True})
|
|
except Failure as exc:
|
|
item['error'] = str(exc)
|
|
except Exception as exc:
|
|
# Never serialize arbitrary library exception text, request or actor.
|
|
item['error'] = type(exc).__name__
|
|
finally:
|
|
with lock:
|
|
item['end_seconds'] = time.monotonic() - origin
|
|
activity['active'] -= 1
|
|
return item, attachment
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=WORKERS) as pool:
|
|
completed = list(pool.map(upload, range(WORKERS)))
|
|
requests = [item for item, _attachment in completed]
|
|
evidence['requests'] = requests
|
|
evidence['elapsed_seconds'] = time.monotonic() - origin
|
|
evidence['peak_active_requests'] = activity['peak']
|
|
overlap = min(item['end_seconds'] for item in requests) - max(item['start_seconds'] for item in requests)
|
|
evidence['all_ten_overlap_seconds'] = max(0, overlap)
|
|
evidence['all_ten_overlapping'] = overlap > 0
|
|
evidence['successful_count'] = sum('error' not in item for item in requests)
|
|
evidence['error_count'] = WORKERS - evidence['successful_count']
|
|
evidence['observed_instances'] = sorted({item['instance'] for item in requests if item.get('instance')})
|
|
successful = [attachment for item, attachment in completed if 'error' not in item]
|
|
if successful:
|
|
download = client.call('GET', '/api/v1/attachments/' + successful[0]['id'] + '/content', actor=bob)
|
|
check(download['bytes'] == payload, 'representative download differs from original')
|
|
evidence['representative_download'] = {
|
|
'verified_as': 'bob (chat member)', 'matches_original_bytes': True,
|
|
'size_bytes': len(download['bytes']), 'sha256': hashlib.sha256(download['bytes']).hexdigest()}
|
|
evidence['final'] = {item['instance']: memory(item['container']) for item in replicas}
|
|
evidence['cluster_after'] = cluster.snapshot()
|
|
states = cluster.inspect(containers)
|
|
evidence['oom_killed'] = {item['Name'].lstrip('/'): item['State']['OOMKilled'] for item in states}
|
|
evidence['per_node_peaks'] = evidence['final']
|
|
check(evidence['successful_count'] == WORKERS, 'not all ten uploads succeeded')
|
|
check(len(evidence['observed_instances']) >= 2, 'uploads did not reach both API instances')
|
|
check(overlap > 0 and activity['peak'] == WORKERS, 'ten requests did not all overlap')
|
|
check(not any(evidence['oom_killed'].values()), 'an API container was OOM-killed')
|
|
for item in replicas:
|
|
before, after = evidence['baseline'][item['instance']], evidence['final'][item['instance']]
|
|
check(after['cgroup_peak_bytes'] <= 512 * 1024 * 1024, 'API exceeded configured memory limit')
|
|
for event in ('oom', 'oom_kill', 'failcnt'):
|
|
check(before['memory_events'].get(event, 0) == 0 and after['memory_events'].get(event, 0) == 0,
|
|
'API recorded an OOM or cgroup allocation failure')
|
|
check(all(item['State']['Running'] and item['State'].get('Health', {}).get('Status') == 'healthy'
|
|
for item in states), 'both APIs must remain running and healthy')
|
|
cluster.wait_cluster(2)
|
|
|
|
|
|
def main():
|
|
evidence = {'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'result': 'FAIL'}
|
|
try:
|
|
experiment(evidence)
|
|
evidence['result'] = 'PASS'
|
|
except Failure as exc:
|
|
evidence['failure'] = str(exc)
|
|
except Exception as exc:
|
|
evidence['failure'] = type(exc).__name__
|
|
finally:
|
|
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
|
|
EVIDENCE.write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + '\n')
|
|
print(evidence['result'] + ': evidence/concurrent-uploads.json')
|
|
if evidence['result'] != 'PASS':
|
|
print(evidence.get('failure', 'experiment failed'))
|
|
raise SystemExit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|