495 lines
12 KiB
Python
495 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
|
||
import hashlib
|
||
import json
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
from collections import Counter
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from pathlib import Path
|
||
|
||
from attachments_negative import make_png
|
||
from client import Client, arguments, check, run
|
||
|
||
|
||
def docker_output(command):
|
||
result = subprocess.run(
|
||
command,
|
||
capture_output=True,
|
||
text=True,
|
||
timeout=10,
|
||
)
|
||
|
||
check(
|
||
result.returncode == 0,
|
||
"Docker command failed while measuring memory",
|
||
)
|
||
|
||
return result.stdout.strip()
|
||
|
||
|
||
def get_api_containers():
|
||
output = docker_output([
|
||
"docker", "compose", "ps", "-q", "api",
|
||
])
|
||
|
||
containers = output.splitlines()
|
||
|
||
check(
|
||
len(containers) == 2,
|
||
"Expected exactly two running API replicas",
|
||
)
|
||
|
||
return containers
|
||
|
||
|
||
def memory_snapshot(container):
|
||
# Docker Desktop с cgroup v2.
|
||
# memory.current включает память всего контейнера,
|
||
# а не только Java heap.
|
||
output = docker_output([
|
||
"docker", "exec", container,
|
||
"sh", "-ec",
|
||
'''
|
||
cat /sys/fs/cgroup/memory.current
|
||
|
||
if [ -r /sys/fs/cgroup/memory.peak ]; then
|
||
cat /sys/fs/cgroup/memory.peak
|
||
else
|
||
echo unavailable
|
||
fi
|
||
|
||
cat /sys/fs/cgroup/memory.events
|
||
''',
|
||
])
|
||
|
||
lines = output.splitlines()
|
||
|
||
check(
|
||
len(lines) >= 2,
|
||
"Cannot read cgroup v2 memory statistics",
|
||
)
|
||
|
||
events = {}
|
||
|
||
for line in lines[2:]:
|
||
key, value = line.split()
|
||
events[key] = int(value)
|
||
|
||
return {
|
||
"current_bytes": int(lines[0]),
|
||
"lifetime_peak_bytes": (
|
||
int(lines[1])
|
||
if lines[1].isdigit()
|
||
else None
|
||
),
|
||
"oom": events.get("oom", 0),
|
||
"oom_kill": events.get("oom_kill", 0),
|
||
}
|
||
|
||
|
||
def mib(value):
|
||
if value is None:
|
||
return None
|
||
|
||
return round(value / 1024 / 1024, 2)
|
||
|
||
|
||
def percentile(values, fraction):
|
||
if not values:
|
||
return None
|
||
|
||
ordered = sorted(values)
|
||
index = int((len(ordered) - 1) * fraction)
|
||
|
||
return round(ordered[index], 2)
|
||
|
||
|
||
def main():
|
||
def configure(parser):
|
||
parser.add_argument("--concurrency", type=int, default=10)
|
||
parser.add_argument("--duration", type=int, default=30)
|
||
parser.add_argument(
|
||
"--output",
|
||
default="results/upload-memory.json",
|
||
)
|
||
|
||
args = arguments(
|
||
"Concurrent image uploads and API memory measurement",
|
||
configure,
|
||
)
|
||
|
||
check(
|
||
args.concurrency >= 10,
|
||
"Use at least 10 concurrent clients",
|
||
)
|
||
|
||
check(
|
||
args.duration >= 10,
|
||
"Duration must be at least 10 seconds",
|
||
)
|
||
|
||
controller = Client(
|
||
args.base_url,
|
||
args.ca_file,
|
||
timeout=40,
|
||
)
|
||
|
||
alice, bob, eve, chat = controller.bootstrap()
|
||
containers = get_api_containers()
|
||
|
||
print("Generating PNG 6000x6000...", flush=True)
|
||
|
||
image = make_png(6000, 6000)
|
||
checksum = hashlib.sha256(image).hexdigest()
|
||
|
||
check(
|
||
len(image) <= 10 * 1024 * 1024,
|
||
"Generated image exceeds upload byte limit",
|
||
)
|
||
|
||
print(
|
||
f"Image: {len(image)} bytes, 36000000 pixels",
|
||
flush=True,
|
||
)
|
||
|
||
path = (
|
||
"/api/v1/chats/"
|
||
+ chat["id"]
|
||
+ "/attachments?filename=memory-test.png"
|
||
)
|
||
|
||
clients = [
|
||
Client(args.base_url, args.ca_file, timeout=40)
|
||
for _ in range(args.concurrency)
|
||
]
|
||
|
||
baseline = {
|
||
container: memory_snapshot(container)
|
||
for container in containers
|
||
}
|
||
|
||
samples = {container: [] for container in containers}
|
||
monitor_errors = []
|
||
stop_monitor = threading.Event()
|
||
|
||
started = time.monotonic()
|
||
|
||
def monitor():
|
||
while not stop_monitor.is_set():
|
||
for container in containers:
|
||
try:
|
||
snapshot = memory_snapshot(container)
|
||
|
||
samples[container].append({
|
||
"elapsed_seconds": round(
|
||
time.monotonic() - started,
|
||
3,
|
||
),
|
||
**snapshot,
|
||
})
|
||
|
||
except Exception as error:
|
||
monitor_errors.append(
|
||
type(error).__name__
|
||
)
|
||
stop_monitor.set()
|
||
return
|
||
|
||
stop_monitor.wait(0.5)
|
||
|
||
barrier = threading.Barrier(args.concurrency + 1)
|
||
|
||
def worker(index):
|
||
client = clients[index]
|
||
|
||
counters = Counter()
|
||
latencies = []
|
||
errors = []
|
||
instances = set()
|
||
successful_instances = set()
|
||
first_success = None
|
||
|
||
barrier.wait()
|
||
|
||
while time.monotonic() - started < args.duration:
|
||
request_started = time.monotonic()
|
||
|
||
try:
|
||
response = client.call(
|
||
"POST",
|
||
path,
|
||
expected=[201, 429],
|
||
actor=alice,
|
||
raw=image,
|
||
headers={"Content-Type": "image/png"},
|
||
schema="Attachment",
|
||
)
|
||
|
||
instance = response["headers"]["X-Instance-Id"]
|
||
instances.add(instance)
|
||
|
||
if response["status"] == 429:
|
||
check(
|
||
response["json"]["error"]["code"]
|
||
== "rate_limited",
|
||
"Unexpected error code for 429",
|
||
)
|
||
|
||
counters["rate_limited"] += 1
|
||
|
||
else:
|
||
attachment = response["json"]
|
||
|
||
check(
|
||
attachment["status"] == "ready",
|
||
"Successful upload is not ready",
|
||
)
|
||
|
||
check(
|
||
attachment["sha256"] == checksum,
|
||
"Uploaded SHA-256 mismatch",
|
||
)
|
||
|
||
check(
|
||
attachment["size_bytes"] == len(image),
|
||
"Uploaded size mismatch",
|
||
)
|
||
|
||
counters["successful"] += 1
|
||
successful_instances.add(instance)
|
||
|
||
latencies.append(
|
||
(time.monotonic() - request_started) * 1000
|
||
)
|
||
|
||
if first_success is None:
|
||
first_success = attachment
|
||
|
||
except Exception as error:
|
||
counters["unexpected_errors"] += 1
|
||
|
||
if len(errors) < 3:
|
||
errors.append(str(error))
|
||
|
||
time.sleep(0.1)
|
||
|
||
return {
|
||
"counters": dict(counters),
|
||
"latencies": latencies,
|
||
"errors": errors,
|
||
"instances": instances,
|
||
"successful_instances": successful_instances,
|
||
"first_success": first_success,
|
||
}
|
||
|
||
monitor_thread = threading.Thread(
|
||
target=monitor,
|
||
daemon=True,
|
||
)
|
||
|
||
with ThreadPoolExecutor(
|
||
max_workers=args.concurrency
|
||
) as executor:
|
||
|
||
futures = [
|
||
executor.submit(worker, index)
|
||
for index in range(args.concurrency)
|
||
]
|
||
|
||
started = time.monotonic()
|
||
monitor_thread.start()
|
||
|
||
print(
|
||
f"Starting {args.concurrency} clients "
|
||
f"for {args.duration} seconds...",
|
||
flush=True,
|
||
)
|
||
|
||
barrier.wait()
|
||
|
||
worker_results = [
|
||
future.result()
|
||
for future in futures
|
||
]
|
||
|
||
elapsed = time.monotonic() - started
|
||
|
||
stop_monitor.set()
|
||
monitor_thread.join(timeout=25)
|
||
|
||
check(
|
||
not monitor_thread.is_alive(),
|
||
"Memory monitor did not stop",
|
||
)
|
||
|
||
final = {
|
||
container: memory_snapshot(container)
|
||
for container in containers
|
||
}
|
||
|
||
totals = Counter()
|
||
latencies = []
|
||
errors = []
|
||
instances = set()
|
||
successful_instances = set()
|
||
|
||
for result in worker_results:
|
||
totals.update(result["counters"])
|
||
latencies.extend(result["latencies"])
|
||
instances.update(result["instances"])
|
||
successful_instances.update(
|
||
result["successful_instances"]
|
||
)
|
||
|
||
errors.extend(result["errors"])
|
||
|
||
download_errors = []
|
||
|
||
for result in worker_results:
|
||
attachment = result["first_success"]
|
||
|
||
if attachment is None:
|
||
continue
|
||
|
||
try:
|
||
response = controller.call(
|
||
"GET",
|
||
"/api/v1/attachments/"
|
||
+ attachment["id"]
|
||
+ "/content",
|
||
actor=bob,
|
||
)
|
||
|
||
check(
|
||
response["bytes"] == image,
|
||
"Downloaded image differs from the original",
|
||
)
|
||
|
||
except Exception as error:
|
||
download_errors.append(str(error))
|
||
|
||
memory_report = {}
|
||
|
||
for container in containers:
|
||
measurements = [
|
||
baseline[container]["current_bytes"],
|
||
final[container]["current_bytes"],
|
||
]
|
||
|
||
measurements.extend(
|
||
sample["current_bytes"]
|
||
for sample in samples[container]
|
||
)
|
||
|
||
memory_report[container[:12]] = {
|
||
"baseline_mib": mib(
|
||
baseline[container]["current_bytes"]
|
||
),
|
||
"observed_peak_mib": mib(max(measurements)),
|
||
"final_mib": mib(
|
||
final[container]["current_bytes"]
|
||
),
|
||
"lifetime_peak_before_mib": mib(
|
||
baseline[container]["lifetime_peak_bytes"]
|
||
),
|
||
"lifetime_peak_after_mib": mib(
|
||
final[container]["lifetime_peak_bytes"]
|
||
),
|
||
"oom_delta": (
|
||
final[container]["oom"]
|
||
- baseline[container]["oom"]
|
||
),
|
||
"oom_kill_delta": (
|
||
final[container]["oom_kill"]
|
||
- baseline[container]["oom_kill"]
|
||
),
|
||
"sample_count": len(samples[container]),
|
||
}
|
||
|
||
success = totals["successful"]
|
||
limited = totals["rate_limited"]
|
||
unexpected = totals["unexpected_errors"]
|
||
|
||
passed = (
|
||
success > 0
|
||
and unexpected == 0
|
||
and not monitor_errors
|
||
and not download_errors
|
||
and len(successful_instances) >= 2
|
||
and all(
|
||
item["sample_count"] > 0
|
||
and item["oom_delta"] == 0
|
||
and item["oom_kill_delta"] == 0
|
||
for item in memory_report.values()
|
||
)
|
||
)
|
||
|
||
report = {
|
||
"passed": passed,
|
||
"workload": "closed-loop image uploads; no automatic retries",
|
||
"concurrency": args.concurrency,
|
||
"requested_duration_seconds": args.duration,
|
||
"elapsed_seconds": round(elapsed, 3),
|
||
"image": {
|
||
"width": 6000,
|
||
"height": 6000,
|
||
"pixels": 36_000_000,
|
||
"size_bytes": len(image),
|
||
"sha256": checksum,
|
||
},
|
||
"successful_uploads": success,
|
||
"rate_limited_requests": limited,
|
||
"unexpected_errors": unexpected,
|
||
"successful_rps": round(success / elapsed, 3),
|
||
"observed_instances": sorted(instances),
|
||
"successful_instances": sorted(successful_instances),
|
||
"successful_latency_ms": {
|
||
"p50": percentile(latencies, 0.50),
|
||
"p95": percentile(latencies, 0.95),
|
||
"p99": percentile(latencies, 0.99),
|
||
},
|
||
"memory": memory_report,
|
||
"error_examples": errors[:10],
|
||
"monitor_errors": monitor_errors,
|
||
"download_errors": download_errors,
|
||
"notes": [
|
||
"429 responses are admission-control rejections, not successful uploads.",
|
||
"Observed memory peak is sampled and may miss short spikes.",
|
||
"Lifetime cgroup peak includes earlier container activity and startup.",
|
||
"Container memory includes more than Java heap.",
|
||
"Concurrency means concurrent clients, not 10 simultaneous decoders.",
|
||
],
|
||
}
|
||
|
||
output = Path(args.output)
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
|
||
saved_report = {
|
||
**report,
|
||
"memory_samples": samples,
|
||
}
|
||
|
||
output.write_text(
|
||
json.dumps(saved_report, ensure_ascii=False, indent=2),
|
||
encoding="utf-8",
|
||
)
|
||
|
||
print(
|
||
json.dumps(report, ensure_ascii=False, indent=2),
|
||
flush=True,
|
||
)
|
||
|
||
print("Report saved to:", output, flush=True)
|
||
|
||
check(
|
||
passed,
|
||
"Experiment detected errors; inspect the saved report",
|
||
)
|
||
|
||
print("PASS: concurrent upload experiment completed", flush=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main) |