Files
mtusi2026lab4/scripts/upload-experiment.py
T

265 lines
9.9 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import datetime
import hashlib
import http.client
import json
import math
import os
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
HOST = os.environ.get('LB_HOST', 'lb')
PORT = int(os.environ.get('LB_PORT', '8080'))
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
MiB = 1024 * 1024
def docker(*args, text=True):
return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout
def replicas():
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def padded_png(size):
return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:]
def big_png(width, height):
rows = (b'\0' + b'\x55' * (width * 3)) * height
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')
def big_jpeg(width, height):
script = (
"require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})"
".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height)
)
return docker('exec', replicas()[0], 'node', '-e', script, text=False)
def cgroup_peak(cid):
raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')
return int(raw.strip()) // MiB
def percentile(values, q):
ordered = sorted(values)
return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3)
class Sampler:
def __init__(self, cids):
self.rss = {cid: 0 for cid in cids}
self.cpu = {}
self.stop_flag = threading.Event()
self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids]
self.threads.append(threading.Thread(target=self.watch_cpu))
def watch(self, cid):
script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done'
process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True)
for line in process.stdout:
if self.stop_flag.is_set():
break
match = re.match(r'VmRSS:\s+(\d+)', line)
if match:
self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024)
process.kill()
process.wait()
def watch_cpu(self):
while not self.stop_flag.is_set():
lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}')
for line in lines.splitlines():
name, cpu, usage = line.split(' ', 2)
value = float(cpu.rstrip('%'))
entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''})
entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value)
entry['last_mem'] = usage.split(' / ')[0]
def __enter__(self):
[thread.start() for thread in self.threads]
return self
def __exit__(self, *_):
self.stop_flag.set()
for thread in self.threads:
thread.join(timeout=5)
def restart_replicas():
docker('compose', 'restart', 'api')
deadline = time.monotonic() + 90
expected = set(replicas())
while time.monotonic() < deadline:
seen = set()
for _ in range(8):
try:
conn = http.client.HTTPConnection(HOST, PORT, timeout=3)
conn.request('GET', '/health/ready')
response = conn.getresponse()
response.read()
if response.status == 200:
seen.add(response.getheader('X-Instance-Id'))
conn.close()
except OSError:
pass
if seen >= expected:
return
time.sleep(0.5)
raise SystemExit('replicas did not return')
def upload(path, token, body, content_type):
conn = http.client.HTTPConnection(HOST, PORT, timeout=120)
started = time.monotonic()
conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type})
response = conn.getresponse()
data = response.read()
elapsed = time.monotonic() - started
conn.close()
return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id')
def download_sha(attachment_id, token):
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token})
response = conn.getresponse()
data = response.read()
conn.close()
return response.status, hashlib.sha256(data).hexdigest()
def run_profile(name, body, content_type, concurrency, rounds, token, chat):
restart_replicas()
cids = replicas()
idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids}
path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin'
expected_sha = hashlib.sha256(body).hexdigest()
results = []
lock = threading.Lock()
def worker():
try:
outcome = upload(path, token, body, content_type)
except OSError as error:
outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None)
with lock:
results.append(outcome)
with Sampler(cids) as sampler:
started = time.monotonic()
for _ in range(rounds):
threads = [threading.Thread(target=worker) for _ in range(concurrency)]
[thread.start() for thread in threads]
[thread.join() for thread in threads]
wall = time.monotonic() - started
time.sleep(0.5)
created = [item for item in results if item[0] == 201]
latencies = [item[1] for item in created]
statuses = {}
for item in results:
statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1
sha_ok = all(item[2]['sha256'] == expected_sha for item in created)
stored_ok = None
if created:
status, sha = download_sha(created[0][2]['id'], token)
stored_ok = status == 200 and sha == expected_sha
per_replica = {}
for cid in cids:
info = docker('exec', cid, 'cat', '/proc/1/status')
hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024
oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip()
temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip()
per_replica[cid] = {
'idle_rss_mib': idle[cid],
'sampled_peak_rss_mib': sampler.rss[cid],
'vm_hwm_mib': hwm,
'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid),
'oom_killed': oom,
'temp_dirs_left': int(temp or 0),
'served': sum(1 for item in results if item[3] == cid),
}
return {
'profile': name,
'body_bytes': len(body),
'content_type': content_type,
'concurrency': concurrency,
'rounds': rounds,
'requests': len(results),
'statuses': statuses,
'wall_seconds': round(wall, 2),
'successful_rps': round(len(created) / wall, 2),
'error_rate': round(1 - len(created) / len(results), 4),
'latency_seconds': None if not latencies else {
'p50': percentile(latencies, 0.5),
'p95': percentile(latencies, 0.95),
'p99': percentile(latencies, 0.99),
},
'sha256_in_responses_ok': sha_ok,
'download_sha256_ok': stored_ok,
'replicas': per_replica,
'docker_stats': sampler.cpu,
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--output')
parser.add_argument('--rounds', type=int, default=3)
args = parser.parse_args()
client = Client('http://%s:%d' % (HOST, PORT))
alice = client.user('upload_load')
chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json']
token, chat_id = alice['token'], chat['id']
jpeg_40mp = big_jpeg(8000, 5000)
png_40mp = big_png(8000, 5000)
profiles = [
('sample 112 KiB x10', SAMPLE, 'image/png', 10),
('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1),
('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10),
('png 40 MP x10', png_40mp, 'image/png', 10),
('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10),
('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20),
('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40),
]
results = []
for name, body, content_type, concurrency in profiles:
print('profile:', name, flush=True)
results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id))
count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip()
limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split()
report = {
'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB},
'rows_after_attachments_messages': count,
'results': results,
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + '\n', encoding='utf-8')
print(text)
if __name__ == '__main__':
main()