forked from mtusi2026/mtusi2026lab4
114 lines
5.5 KiB
Python
114 lines
5.5 KiB
Python
"""Повторяемые параллельные upload и память cgroup API, без записи токенов."""
|
|
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
from io import BytesIO
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import subprocess
|
|
from threading import Barrier
|
|
import time
|
|
|
|
from PIL import Image
|
|
|
|
from scripts.deployment import ComposeProject
|
|
from tests.client import Client, Failure, arguments, check, run
|
|
|
|
MONITOR = r'''
|
|
import json, pathlib, sys, threading
|
|
root = pathlib.Path('/sys/fs/cgroup')
|
|
def current():
|
|
return int((root / 'memory.current').read_text())
|
|
def peak():
|
|
path = root / 'memory.peak'
|
|
return int(path.read_text()) if path.exists() else None
|
|
stop = threading.Event()
|
|
threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start()
|
|
initial, before = current(), peak()
|
|
sampled = initial
|
|
print('ready', flush=True)
|
|
while not stop.wait(0.02):
|
|
sampled = max(sampled, current())
|
|
print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled,
|
|
'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True)
|
|
'''
|
|
|
|
|
|
def main():
|
|
def configure(parser):
|
|
parser.add_argument('--concurrency', type=int, default=10)
|
|
parser.add_argument('--rounds', type=int, default=3)
|
|
parser.add_argument('--output', type=Path)
|
|
args = arguments('Concurrent attachment uploads and API memory', configure)
|
|
check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10')
|
|
project = ComposeProject()
|
|
client = Client(args.base_url, args.ca_file, timeout=65)
|
|
alice, _, _, chat = client.bootstrap()
|
|
random_bytes = random.Random(42).randbytes(1500 * 1500 * 3)
|
|
image = Image.frombytes('RGB', (1500, 1500), random_bytes)
|
|
buffer = BytesIO()
|
|
image.save(buffer, format='PNG')
|
|
data = buffer.getvalue()
|
|
check(len(data) <= 10485760, 'fixture exceeds upload limit')
|
|
monitors = []
|
|
measurements = {}
|
|
rows = []
|
|
started = time.monotonic()
|
|
try:
|
|
for replica in project.replicas():
|
|
process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR],
|
|
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
monitors.append((replica.instance_id, process))
|
|
check(process.stdout.readline().strip() == 'ready', 'memory monitor failed')
|
|
check(bool(monitors), 'API not running')
|
|
for _ in range(args.rounds):
|
|
barrier = Barrier(args.concurrency)
|
|
def upload(index):
|
|
local = Client(args.base_url, args.ca_file, timeout=65)
|
|
barrier.wait(timeout=15)
|
|
request_started = time.monotonic()
|
|
try:
|
|
return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png",
|
|
expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data,
|
|
headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
except Failure:
|
|
# При раннем отказе сервер может закрыть ещё отправляемое тело.
|
|
return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started}
|
|
with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
|
rows.extend(pool.map(upload, range(args.concurrency)))
|
|
finally:
|
|
elapsed = time.monotonic() - started
|
|
for alias, process in monitors:
|
|
output, _ = process.communicate(input='', timeout=10)
|
|
check(process.returncode == 0, 'memory monitor failed')
|
|
measurements[alias] = json.loads(output)
|
|
succeeded = [row for row in rows if row['status'] == 201]
|
|
check(bool(succeeded), 'all uploads were rejected')
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
for row in succeeded:
|
|
check(row['json']['sha256'] == digest, 'upload checksum differs')
|
|
original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes']
|
|
check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs')
|
|
latencies = sorted(row['elapsed'] for row in rows)
|
|
result = {
|
|
'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows),
|
|
'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest},
|
|
'statuses': dict(Counter(str(row['status']) for row in rows)),
|
|
'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded),
|
|
'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3),
|
|
'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'],
|
|
'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries',
|
|
}
|
|
encoded = json.dumps(result, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(encoded + '\n')
|
|
print(encoded)
|
|
check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|