forked from mtusi2026/mtusi2026lab4
265 lines
9.9 KiB
Python
Executable File
265 lines
9.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
HOST = os.environ.get('LB_HOST', 'lb')
|
|
PORT = int(os.environ.get('LB_PORT', '8080'))
|
|
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
MiB = 1024 * 1024
|
|
|
|
|
|
def docker(*args, text=True):
|
|
return subprocess.run(['docker', *args], check=True, capture_output=True, text=text).stdout
|
|
|
|
|
|
def replicas():
|
|
return [cid[:12] for cid in docker('compose', 'ps', '-q', 'api').split()]
|
|
|
|
|
|
def chunk(kind, data):
|
|
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
|
|
|
|
|
def padded_png(size):
|
|
return SAMPLE[:33] + chunk(b'abCd', b'\x01' * (size - len(SAMPLE) - 12)) + SAMPLE[33:]
|
|
|
|
|
|
def big_png(width, height):
|
|
rows = (b'\0' + b'\x55' * (width * 3)) * height
|
|
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
|
return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', header) + chunk(b'IDAT', zlib.compress(rows, 9)) + chunk(b'IEND', b'')
|
|
|
|
|
|
def big_jpeg(width, height):
|
|
script = (
|
|
"require('sharp')({create:{width:%d,height:%d,channels:3,background:'#336699'}})"
|
|
".jpeg({quality:80}).toBuffer().then(b=>process.stdout.write(b))" % (width, height)
|
|
)
|
|
return docker('exec', replicas()[0], 'node', '-e', script, text=False)
|
|
|
|
|
|
def cgroup_peak(cid):
|
|
raw = docker('exec', cid, 'sh', '-c', 'cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes 2>/dev/null || cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 0')
|
|
return int(raw.strip()) // MiB
|
|
|
|
|
|
def percentile(values, q):
|
|
ordered = sorted(values)
|
|
return round(ordered[max(0, math.ceil(q * len(ordered)) - 1)], 3)
|
|
|
|
|
|
class Sampler:
|
|
def __init__(self, cids):
|
|
self.rss = {cid: 0 for cid in cids}
|
|
self.cpu = {}
|
|
self.stop_flag = threading.Event()
|
|
self.threads = [threading.Thread(target=self.watch, args=(cid,)) for cid in cids]
|
|
self.threads.append(threading.Thread(target=self.watch_cpu))
|
|
|
|
def watch(self, cid):
|
|
script = 'while :; do grep VmRSS /proc/1/status; sleep 0.2; done'
|
|
process = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', script], stdout=subprocess.PIPE, text=True)
|
|
for line in process.stdout:
|
|
if self.stop_flag.is_set():
|
|
break
|
|
match = re.match(r'VmRSS:\s+(\d+)', line)
|
|
if match:
|
|
self.rss[cid] = max(self.rss[cid], int(match.group(1)) // 1024)
|
|
process.kill()
|
|
process.wait()
|
|
|
|
def watch_cpu(self):
|
|
while not self.stop_flag.is_set():
|
|
lines = docker('stats', '--no-stream', '--format', '{{.Name}} {{.CPUPerc}} {{.MemUsage}}')
|
|
for line in lines.splitlines():
|
|
name, cpu, usage = line.split(' ', 2)
|
|
value = float(cpu.rstrip('%'))
|
|
entry = self.cpu.setdefault(name, {'max_cpu_percent': 0, 'last_mem': ''})
|
|
entry['max_cpu_percent'] = max(entry['max_cpu_percent'], value)
|
|
entry['last_mem'] = usage.split(' / ')[0]
|
|
|
|
def __enter__(self):
|
|
[thread.start() for thread in self.threads]
|
|
return self
|
|
|
|
def __exit__(self, *_):
|
|
self.stop_flag.set()
|
|
for thread in self.threads:
|
|
thread.join(timeout=5)
|
|
|
|
|
|
def restart_replicas():
|
|
docker('compose', 'restart', 'api')
|
|
deadline = time.monotonic() + 90
|
|
expected = set(replicas())
|
|
while time.monotonic() < deadline:
|
|
seen = set()
|
|
for _ in range(8):
|
|
try:
|
|
conn = http.client.HTTPConnection(HOST, PORT, timeout=3)
|
|
conn.request('GET', '/health/ready')
|
|
response = conn.getresponse()
|
|
response.read()
|
|
if response.status == 200:
|
|
seen.add(response.getheader('X-Instance-Id'))
|
|
conn.close()
|
|
except OSError:
|
|
pass
|
|
if seen >= expected:
|
|
return
|
|
time.sleep(0.5)
|
|
raise SystemExit('replicas did not return')
|
|
|
|
|
|
def upload(path, token, body, content_type):
|
|
conn = http.client.HTTPConnection(HOST, PORT, timeout=120)
|
|
started = time.monotonic()
|
|
conn.request('POST', path, body=body, headers={'Authorization': 'Bearer ' + token, 'Content-Type': content_type})
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
elapsed = time.monotonic() - started
|
|
conn.close()
|
|
return response.status, elapsed, json.loads(data), response.getheader('X-Instance-Id')
|
|
|
|
|
|
def download_sha(attachment_id, token):
|
|
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
|
|
conn.request('GET', '/api/v1/attachments/' + attachment_id + '/content', headers={'Authorization': 'Bearer ' + token})
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
conn.close()
|
|
return response.status, hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def run_profile(name, body, content_type, concurrency, rounds, token, chat):
|
|
restart_replicas()
|
|
cids = replicas()
|
|
idle = {cid: int(re.search(r'VmRSS:\s+(\d+)', docker('exec', cid, 'cat', '/proc/1/status')).group(1)) // 1024 for cid in cids}
|
|
path = '/api/v1/chats/' + chat + '/attachments?filename=load.bin'
|
|
expected_sha = hashlib.sha256(body).hexdigest()
|
|
results = []
|
|
lock = threading.Lock()
|
|
|
|
def worker():
|
|
try:
|
|
outcome = upload(path, token, body, content_type)
|
|
except OSError as error:
|
|
outcome = (0, 0.0, {'error': {'code': type(error).__name__}}, None)
|
|
with lock:
|
|
results.append(outcome)
|
|
|
|
with Sampler(cids) as sampler:
|
|
started = time.monotonic()
|
|
for _ in range(rounds):
|
|
threads = [threading.Thread(target=worker) for _ in range(concurrency)]
|
|
[thread.start() for thread in threads]
|
|
[thread.join() for thread in threads]
|
|
wall = time.monotonic() - started
|
|
time.sleep(0.5)
|
|
|
|
created = [item for item in results if item[0] == 201]
|
|
latencies = [item[1] for item in created]
|
|
statuses = {}
|
|
for item in results:
|
|
statuses[str(item[0])] = statuses.get(str(item[0]), 0) + 1
|
|
sha_ok = all(item[2]['sha256'] == expected_sha for item in created)
|
|
stored_ok = None
|
|
if created:
|
|
status, sha = download_sha(created[0][2]['id'], token)
|
|
stored_ok = status == 200 and sha == expected_sha
|
|
per_replica = {}
|
|
for cid in cids:
|
|
info = docker('exec', cid, 'cat', '/proc/1/status')
|
|
hwm = int(re.search(r'VmHWM:\s+(\d+)', info).group(1)) // 1024
|
|
oom = docker('inspect', '-f', '{{.State.OOMKilled}}', cid).strip()
|
|
temp = docker('exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true').strip()
|
|
per_replica[cid] = {
|
|
'idle_rss_mib': idle[cid],
|
|
'sampled_peak_rss_mib': sampler.rss[cid],
|
|
'vm_hwm_mib': hwm,
|
|
'cgroup_max_usage_with_page_cache_mib': cgroup_peak(cid),
|
|
'oom_killed': oom,
|
|
'temp_dirs_left': int(temp or 0),
|
|
'served': sum(1 for item in results if item[3] == cid),
|
|
}
|
|
return {
|
|
'profile': name,
|
|
'body_bytes': len(body),
|
|
'content_type': content_type,
|
|
'concurrency': concurrency,
|
|
'rounds': rounds,
|
|
'requests': len(results),
|
|
'statuses': statuses,
|
|
'wall_seconds': round(wall, 2),
|
|
'successful_rps': round(len(created) / wall, 2),
|
|
'error_rate': round(1 - len(created) / len(results), 4),
|
|
'latency_seconds': None if not latencies else {
|
|
'p50': percentile(latencies, 0.5),
|
|
'p95': percentile(latencies, 0.95),
|
|
'p99': percentile(latencies, 0.99),
|
|
},
|
|
'sha256_in_responses_ok': sha_ok,
|
|
'download_sha256_ok': stored_ok,
|
|
'replicas': per_replica,
|
|
'docker_stats': sampler.cpu,
|
|
}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--output')
|
|
parser.add_argument('--rounds', type=int, default=3)
|
|
args = parser.parse_args()
|
|
client = Client('http://%s:%d' % (HOST, PORT))
|
|
alice = client.user('upload_load')
|
|
chat = client.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'upload experiment', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
|
token, chat_id = alice['token'], chat['id']
|
|
jpeg_40mp = big_jpeg(8000, 5000)
|
|
png_40mp = big_png(8000, 5000)
|
|
profiles = [
|
|
('sample 112 KiB x10', SAMPLE, 'image/png', 10),
|
|
('png 10 MiB x1', padded_png(10 * MiB), 'image/png', 1),
|
|
('png 10 MiB x10', padded_png(10 * MiB), 'image/png', 10),
|
|
('png 40 MP x10', png_40mp, 'image/png', 10),
|
|
('jpeg 40 MP x10', jpeg_40mp, 'image/jpeg', 10),
|
|
('png 10 MiB x20', padded_png(10 * MiB), 'image/png', 20),
|
|
('png 10 MiB x40 (выше лимита 16 на реплику)', padded_png(10 * MiB), 'image/png', 40),
|
|
]
|
|
results = []
|
|
for name, body, content_type, concurrency in profiles:
|
|
print('profile:', name, flush=True)
|
|
results.append(run_profile(name, body, content_type, concurrency, args.rounds, token, chat_id))
|
|
count = docker('compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', 'SELECT (SELECT count(*) FROM attachments), (SELECT count(*) FROM messages)').strip()
|
|
limits = docker('inspect', '-f', '{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}', replicas()[0]).split()
|
|
report = {
|
|
'date': datetime.datetime.now(datetime.timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'),
|
|
'api_replica_limits': {'cpus': int(limits[0]) / 1e9, 'memory_mib': int(limits[1]) // MiB},
|
|
'rows_after_attachments_messages': count,
|
|
'results': results,
|
|
}
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
|
print(text)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|