#!/usr/bin/env python3 import hashlib import json import subprocess import threading import time from collections import Counter from concurrent.futures import ThreadPoolExecutor from pathlib import Path from attachments_negative import make_png from client import Client, arguments, check, run def docker_output(command): result = subprocess.run( command, capture_output=True, text=True, timeout=10, ) check( result.returncode == 0, "Docker command failed while measuring memory", ) return result.stdout.strip() def get_api_containers(): output = docker_output([ "docker", "compose", "ps", "-q", "api", ]) containers = output.splitlines() check( len(containers) == 2, "Expected exactly two running API replicas", ) return containers def memory_snapshot(container): # Docker Desktop с cgroup v2. # memory.current включает память всего контейнера, # а не только Java heap. output = docker_output([ "docker", "exec", container, "sh", "-ec", ''' cat /sys/fs/cgroup/memory.current if [ -r /sys/fs/cgroup/memory.peak ]; then cat /sys/fs/cgroup/memory.peak else echo unavailable fi cat /sys/fs/cgroup/memory.events ''', ]) lines = output.splitlines() check( len(lines) >= 2, "Cannot read cgroup v2 memory statistics", ) events = {} for line in lines[2:]: key, value = line.split() events[key] = int(value) return { "current_bytes": int(lines[0]), "lifetime_peak_bytes": ( int(lines[1]) if lines[1].isdigit() else None ), "oom": events.get("oom", 0), "oom_kill": events.get("oom_kill", 0), } def mib(value): if value is None: return None return round(value / 1024 / 1024, 2) def percentile(values, fraction): if not values: return None ordered = sorted(values) index = int((len(ordered) - 1) * fraction) return round(ordered[index], 2) def main(): def configure(parser): parser.add_argument("--concurrency", type=int, default=10) parser.add_argument("--duration", type=int, default=30) parser.add_argument( "--output", default="results/upload-memory.json", ) args = arguments( "Concurrent image uploads and API memory measurement", configure, ) check( args.concurrency >= 10, "Use at least 10 concurrent clients", ) check( args.duration >= 10, "Duration must be at least 10 seconds", ) controller = Client( args.base_url, args.ca_file, timeout=40, ) alice, bob, eve, chat = controller.bootstrap() containers = get_api_containers() print("Generating PNG 6000x6000...", flush=True) image = make_png(6000, 6000) checksum = hashlib.sha256(image).hexdigest() check( len(image) <= 10 * 1024 * 1024, "Generated image exceeds upload byte limit", ) print( f"Image: {len(image)} bytes, 36000000 pixels", flush=True, ) path = ( "/api/v1/chats/" + chat["id"] + "/attachments?filename=memory-test.png" ) clients = [ Client(args.base_url, args.ca_file, timeout=40) for _ in range(args.concurrency) ] baseline = { container: memory_snapshot(container) for container in containers } samples = {container: [] for container in containers} monitor_errors = [] stop_monitor = threading.Event() started = time.monotonic() def monitor(): while not stop_monitor.is_set(): for container in containers: try: snapshot = memory_snapshot(container) samples[container].append({ "elapsed_seconds": round( time.monotonic() - started, 3, ), **snapshot, }) except Exception as error: monitor_errors.append( type(error).__name__ ) stop_monitor.set() return stop_monitor.wait(0.5) barrier = threading.Barrier(args.concurrency + 1) def worker(index): client = clients[index] counters = Counter() latencies = [] errors = [] instances = set() successful_instances = set() first_success = None barrier.wait() while time.monotonic() - started < args.duration: request_started = time.monotonic() try: response = client.call( "POST", path, expected=[201, 429], actor=alice, raw=image, headers={"Content-Type": "image/png"}, schema="Attachment", ) instance = response["headers"]["X-Instance-Id"] instances.add(instance) if response["status"] == 429: check( response["json"]["error"]["code"] == "rate_limited", "Unexpected error code for 429", ) counters["rate_limited"] += 1 else: attachment = response["json"] check( attachment["status"] == "ready", "Successful upload is not ready", ) check( attachment["sha256"] == checksum, "Uploaded SHA-256 mismatch", ) check( attachment["size_bytes"] == len(image), "Uploaded size mismatch", ) counters["successful"] += 1 successful_instances.add(instance) latencies.append( (time.monotonic() - request_started) * 1000 ) if first_success is None: first_success = attachment except Exception as error: counters["unexpected_errors"] += 1 if len(errors) < 3: errors.append(str(error)) time.sleep(0.1) return { "counters": dict(counters), "latencies": latencies, "errors": errors, "instances": instances, "successful_instances": successful_instances, "first_success": first_success, } monitor_thread = threading.Thread( target=monitor, daemon=True, ) with ThreadPoolExecutor( max_workers=args.concurrency ) as executor: futures = [ executor.submit(worker, index) for index in range(args.concurrency) ] started = time.monotonic() monitor_thread.start() print( f"Starting {args.concurrency} clients " f"for {args.duration} seconds...", flush=True, ) barrier.wait() worker_results = [ future.result() for future in futures ] elapsed = time.monotonic() - started stop_monitor.set() monitor_thread.join(timeout=25) check( not monitor_thread.is_alive(), "Memory monitor did not stop", ) final = { container: memory_snapshot(container) for container in containers } totals = Counter() latencies = [] errors = [] instances = set() successful_instances = set() for result in worker_results: totals.update(result["counters"]) latencies.extend(result["latencies"]) instances.update(result["instances"]) successful_instances.update( result["successful_instances"] ) errors.extend(result["errors"]) download_errors = [] for result in worker_results: attachment = result["first_success"] if attachment is None: continue try: response = controller.call( "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob, ) check( response["bytes"] == image, "Downloaded image differs from the original", ) except Exception as error: download_errors.append(str(error)) memory_report = {} for container in containers: measurements = [ baseline[container]["current_bytes"], final[container]["current_bytes"], ] measurements.extend( sample["current_bytes"] for sample in samples[container] ) memory_report[container[:12]] = { "baseline_mib": mib( baseline[container]["current_bytes"] ), "observed_peak_mib": mib(max(measurements)), "final_mib": mib( final[container]["current_bytes"] ), "lifetime_peak_before_mib": mib( baseline[container]["lifetime_peak_bytes"] ), "lifetime_peak_after_mib": mib( final[container]["lifetime_peak_bytes"] ), "oom_delta": ( final[container]["oom"] - baseline[container]["oom"] ), "oom_kill_delta": ( final[container]["oom_kill"] - baseline[container]["oom_kill"] ), "sample_count": len(samples[container]), } success = totals["successful"] limited = totals["rate_limited"] unexpected = totals["unexpected_errors"] passed = ( success > 0 and unexpected == 0 and not monitor_errors and not download_errors and len(successful_instances) >= 2 and all( item["sample_count"] > 0 and item["oom_delta"] == 0 and item["oom_kill_delta"] == 0 for item in memory_report.values() ) ) report = { "passed": passed, "workload": "closed-loop image uploads; no automatic retries", "concurrency": args.concurrency, "requested_duration_seconds": args.duration, "elapsed_seconds": round(elapsed, 3), "image": { "width": 6000, "height": 6000, "pixels": 36_000_000, "size_bytes": len(image), "sha256": checksum, }, "successful_uploads": success, "rate_limited_requests": limited, "unexpected_errors": unexpected, "successful_rps": round(success / elapsed, 3), "observed_instances": sorted(instances), "successful_instances": sorted(successful_instances), "successful_latency_ms": { "p50": percentile(latencies, 0.50), "p95": percentile(latencies, 0.95), "p99": percentile(latencies, 0.99), }, "memory": memory_report, "error_examples": errors[:10], "monitor_errors": monitor_errors, "download_errors": download_errors, "notes": [ "429 responses are admission-control rejections, not successful uploads.", "Observed memory peak is sampled and may miss short spikes.", "Lifetime cgroup peak includes earlier container activity and startup.", "Container memory includes more than Java heap.", "Concurrency means concurrent clients, not 10 simultaneous decoders.", ], } output = Path(args.output) output.parent.mkdir(parents=True, exist_ok=True) saved_report = { **report, "memory_samples": samples, } output.write_text( json.dumps(saved_report, ensure_ascii=False, indent=2), encoding="utf-8", ) print( json.dumps(report, ensure_ascii=False, indent=2), flush=True, ) print("Report saved to:", output, flush=True) check( passed, "Experiment detected errors; inspect the saved report", ) print("PASS: concurrent upload experiment completed", flush=True) if __name__ == "__main__": run(main)