Files
mtusi2026lab5-fork/scripts/upload_load.py
T
2026-11-07 16:10:32 +03:00

114 lines
5.5 KiB
Python

"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import hashlib
from io import BytesIO
import json
from pathlib import Path
import random
import subprocess
from threading import Barrier
import time
from PIL import Image
from scripts.deployment import ComposeProject
from tests.client import Client, Failure, arguments, check, run
MONITOR = r'''
import json, pathlib, sys, threading
root = pathlib.Path('/sys/fs/cgroup')
def current():
return int((root / 'memory.current').read_text())
def peak():
path = root / 'memory.peak'
return int(path.read_text()) if path.exists() else None
stop = threading.Event()
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
initial, before = current(), peak()
sampled = initial
print('ready', flush=True)
while not stop.wait(0.02):
sampled = max(sampled, current())
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
'''
def main():
def configure(parser):
parser.add_argument('--concurrency', type=int, default=10)
parser.add_argument('--rounds', type=int, default=3)
parser.add_argument('--output', type=Path)
args = arguments('Concurrent attachment uploads and API memory', configure)
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
project = ComposeProject()
client = Client(args.base_url, args.ca_file, timeout=65)
alice, _, _, chat = client.bootstrap()
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
buffer = BytesIO()
image.save(buffer, format='PNG')
data = buffer.getvalue()
check(len(data) <= 10485760, 'fixture exceeds upload limit')
monitors = []
measurements = {}
rows = []
started = time.monotonic()
try:
for replica in project.replicas():
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
monitors.append((replica.instance_id, process))
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
check(bool(monitors), 'API not running')
for _ in range(args.rounds):
barrier = Barrier(args.concurrency)
def upload(index):
local = Client(args.base_url, args.ca_file, timeout=65)
barrier.wait(timeout=15)
request_started = time.monotonic()
try:
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
except Failure:
# При раннем отказе сервер может закрыть ещё отправляемое тело.
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
rows.extend(pool.map(upload, range(args.concurrency)))
finally:
elapsed = time.monotonic() - started
for alias, process in monitors:
output, _ = process.communicate(input='', timeout=10)
check(process.returncode == 0, 'memory monitor failed')
measurements[alias] = json.loads(output)
succeeded = [row for row in rows if row['status'] == 201]
check(bool(succeeded), 'all uploads were rejected')
digest = hashlib.sha256(data).hexdigest()
for row in succeeded:
check(row['json']['sha256'] == digest, 'upload checksum differs')
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
latencies = sorted(row['elapsed'] for row in rows)
result = {
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
'statuses': dict(Counter(str(row['status']) for row in rows)),
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
}
encoded = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(encoded + '\n')
print(encoded)
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
if __name__ == '__main__':
run(main)