130 lines
4.7 KiB
Python
Executable File
130 lines
4.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Concurrent upload experiment with API memory measurement; creates synthetic data."""
|
|
import json
|
|
import math
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
from client import Client, Failure, arguments, check, run
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def engine():
|
|
return 'podman' if shutil.which('podman') else 'docker'
|
|
|
|
|
|
def api_containers(project):
|
|
result = subprocess.run(
|
|
[engine(), 'ps',
|
|
'--filter', 'label=com.docker.compose.project=' + project,
|
|
'--filter', 'label=com.docker.compose.service=api',
|
|
'--format', '{{.Names}}'],
|
|
capture_output=True, text=True, check=False)
|
|
check(result.returncode == 0, 'cannot list api containers: ' + result.stderr.strip())
|
|
return sorted(line.strip() for line in result.stdout.splitlines() if line.strip())
|
|
|
|
|
|
def peak_bytes(container):
|
|
for path in ('/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes'):
|
|
result = subprocess.run([engine(), 'exec', container, 'cat', path], capture_output=True, text=True, check=False)
|
|
if result.returncode == 0 and result.stdout.strip().isdigit():
|
|
return int(result.stdout.strip())
|
|
return None
|
|
|
|
|
|
def main():
|
|
def configure(p):
|
|
p.add_argument('--concurrency', type=int, default=10)
|
|
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/large.png'))
|
|
p.add_argument('--project', default='messenger-lab4')
|
|
p.add_argument('--output', default=None)
|
|
|
|
args = arguments('Concurrent attachment uploads with peak memory measurement', configure)
|
|
check(1 <= args.concurrency <= 64, 'concurrency must be 1..64')
|
|
source = Path(args.file)
|
|
check(source.is_file(), 'upload fixture not found: ' + str(source))
|
|
content = source.read_bytes()
|
|
|
|
containers = api_containers(args.project)
|
|
check(bool(containers), 'no api containers found for project ' + args.project)
|
|
baseline = {name: peak_bytes(name) for name in containers}
|
|
|
|
c = Client()
|
|
alice, _, _, chat = c.bootstrap()
|
|
actor = {'id': alice['id'], 'token': alice['token']}
|
|
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + source.name
|
|
|
|
barrier = threading.Barrier(args.concurrency)
|
|
lock = threading.Lock()
|
|
latencies, failures = [], []
|
|
|
|
def worker(index):
|
|
local = Client(timeout=120)
|
|
barrier.wait()
|
|
started = time.monotonic()
|
|
try:
|
|
response = local.call('POST', path, 201, actor=actor, raw=content,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
with lock:
|
|
latencies.append(time.monotonic() - started)
|
|
check(response['json']['size_bytes'] == len(content), 'uploaded size mismatch')
|
|
except Failure as exc:
|
|
with lock:
|
|
failures.append(str(exc))
|
|
|
|
started = time.monotonic()
|
|
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
elapsed = time.monotonic() - started
|
|
|
|
peaks = {name: peak_bytes(name) for name in containers}
|
|
peak_delta = {}
|
|
for name in containers:
|
|
before = baseline[name]
|
|
after = peaks[name]
|
|
if before is None or after is None:
|
|
peak_delta[name] = None
|
|
else:
|
|
peak_delta[name] = after - before
|
|
|
|
ordered = sorted(latencies)
|
|
def percentile(p):
|
|
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) if ordered else None
|
|
|
|
report = {
|
|
'concurrency': args.concurrency,
|
|
'uploads': args.concurrency,
|
|
'fixture': source.name,
|
|
'fixture_bytes': len(content),
|
|
'elapsed_seconds': round(elapsed, 3),
|
|
'successful': len(latencies),
|
|
'failed': len(failures),
|
|
'failures': failures[:3],
|
|
'latency_ms': {'p50': percentile(0.50), 'p95': percentile(0.95), 'p99': percentile(0.99)},
|
|
'api_containers': containers,
|
|
'peak_bytes_before': baseline,
|
|
'peak_bytes_after': peaks,
|
|
'peak_delta_bytes': peak_delta,
|
|
'peak_max_mb': round(max(v for v in peaks.values() if v is not None) / (1024 * 1024), 2) if any(v is not None for v in peaks.values()) else None,
|
|
}
|
|
text = json.dumps(report, indent=2)
|
|
if args.output:
|
|
destination = Path(args.output)
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
destination.write_text(text + '\n')
|
|
print(text)
|
|
check(not failures, 'some uploads failed')
|
|
check(len(latencies) == args.concurrency, 'not all uploads succeeded')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main) |