"""Повторяемые параллельные upload и память cgroup API, без записи токенов.""" from collections import Counter from concurrent.futures import ThreadPoolExecutor import hashlib from io import BytesIO import json from pathlib import Path import random import subprocess from threading import Barrier import time from PIL import Image from scripts.deployment import ComposeProject from tests.client import Client, Failure, arguments, check, run MONITOR = r''' import json, pathlib, sys, threading root = pathlib.Path('/sys/fs/cgroup') def current(): return int((root / 'memory.current').read_text()) def peak(): path = root / 'memory.peak' return int(path.read_text()) if path.exists() else None stop = threading.Event() threading.Thread(target=lambda: (sys.stdin.read(), stop.set()), daemon=True).start() initial, before = current(), peak() sampled = initial print('ready', flush=True) while not stop.wait(0.02): sampled = max(sampled, current()) print(json.dumps({'initial_bytes': initial, 'sampled_peak_bytes': sampled, 'cgroup_peak_before_bytes': before, 'cgroup_peak_after_bytes': peak(), 'interval_ms': 20}), flush=True) ''' def main(): def configure(parser): parser.add_argument('--concurrency', type=int, default=10) parser.add_argument('--rounds', type=int, default=3) parser.add_argument('--output', type=Path) args = arguments('Concurrent attachment uploads and API memory', configure) check(10 <= args.concurrency <= 32 and 1 <= args.rounds <= 10, 'use concurrency 10..32 and rounds 1..10') project = ComposeProject() client = Client(args.base_url, args.ca_file, timeout=65) alice, _, _, chat = client.bootstrap() random_bytes = random.Random(42).randbytes(1500 * 1500 * 3) image = Image.frombytes('RGB', (1500, 1500), random_bytes) buffer = BytesIO() image.save(buffer, format='PNG') data = buffer.getvalue() check(len(data) <= 10485760, 'fixture exceeds upload limit') monitors = [] measurements = {} rows = [] started = time.monotonic() try: for replica in project.replicas(): process = subprocess.Popen(['docker', 'exec', '-i', replica.id, 'python', '-c', MONITOR], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) monitors.append((replica.instance_id, process)) check(process.stdout.readline().strip() == 'ready', 'memory monitor failed') check(bool(monitors), 'API not running') for _ in range(args.rounds): barrier = Barrier(args.concurrency) def upload(index): local = Client(args.base_url, args.ca_file, timeout=65) barrier.wait(timeout=15) request_started = time.monotonic() try: return local.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=load.png", expected=[201, 400, 401, 404, 409, 413, 415, 429, 500, 502, 503, 504], actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') except Failure: # При раннем отказе сервер может закрыть ещё отправляемое тело. return {'status': 'failed_request', 'elapsed': time.monotonic() - request_started} with ThreadPoolExecutor(max_workers=args.concurrency) as pool: rows.extend(pool.map(upload, range(args.concurrency))) finally: elapsed = time.monotonic() - started for alias, process in monitors: output, _ = process.communicate(input='', timeout=10) check(process.returncode == 0, 'memory monitor failed') measurements[alias] = json.loads(output) succeeded = [row for row in rows if row['status'] == 201] check(bool(succeeded), 'all uploads were rejected') digest = hashlib.sha256(data).hexdigest() for row in succeeded: check(row['json']['sha256'] == digest, 'upload checksum differs') original = client.call('GET', '/api/v1/attachments/' + row['json']['id'] + '/content', actor=alice)['bytes'] check(hashlib.sha256(original).hexdigest() == digest, 'download checksum differs') latencies = sorted(row['elapsed'] for row in rows) result = { 'concurrency': args.concurrency, 'rounds': args.rounds, 'attempts': len(rows), 'fixture': {'format': 'PNG', 'width': 1500, 'height': 1500, 'size_bytes': len(data), 'sha256': digest}, 'statuses': dict(Counter(str(row['status']) for row in rows)), 'elapsed_seconds': round(elapsed, 3), 'successful_uploads': len(succeeded), 'p95_seconds': round(latencies[min(len(latencies) - 1, int(len(latencies) * .95))], 3), 'memory': measurements, 'api_memory_limit': project.config['services']['api']['deploy']['resources']['limits']['memory'], 'notes': '20 ms samples include monitor and decoder processes; cgroup peak is cumulative since container creation; 429 and failed requests counted separately; no upload retries', } encoded = json.dumps(result, ensure_ascii=False, indent=2) if args.output: args.output.parent.mkdir(parents=True, exist_ok=True) args.output.write_text(encoded + '\n') print(encoded) check(all(row['status'] in {201, 429} for row in rows), 'unexpected HTTP/transport errors during upload experiment; inspect status counts') if __name__ == '__main__': run(main)