Files
mtusi2026lab4/tests/upload_memory_check.py
T
2026-11-08 18:17:48 +03:00

495 lines
12 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
import hashlib
import json
import subprocess
import threading
import time
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from attachments_negative import make_png
from client import Client, arguments, check, run
def docker_output(command):
result = subprocess.run(
command,
capture_output=True,
text=True,
timeout=10,
)
check(
result.returncode == 0,
"Docker command failed while measuring memory",
)
return result.stdout.strip()
def get_api_containers():
output = docker_output([
"docker", "compose", "ps", "-q", "api",
])
containers = output.splitlines()
check(
len(containers) == 2,
"Expected exactly two running API replicas",
)
return containers
def memory_snapshot(container):
# Docker Desktop с cgroup v2.
# memory.current включает память всего контейнера,
# а не только Java heap.
output = docker_output([
"docker", "exec", container,
"sh", "-ec",
'''
cat /sys/fs/cgroup/memory.current
if [ -r /sys/fs/cgroup/memory.peak ]; then
cat /sys/fs/cgroup/memory.peak
else
echo unavailable
fi
cat /sys/fs/cgroup/memory.events
''',
])
lines = output.splitlines()
check(
len(lines) >= 2,
"Cannot read cgroup v2 memory statistics",
)
events = {}
for line in lines[2:]:
key, value = line.split()
events[key] = int(value)
return {
"current_bytes": int(lines[0]),
"lifetime_peak_bytes": (
int(lines[1])
if lines[1].isdigit()
else None
),
"oom": events.get("oom", 0),
"oom_kill": events.get("oom_kill", 0),
}
def mib(value):
if value is None:
return None
return round(value / 1024 / 1024, 2)
def percentile(values, fraction):
if not values:
return None
ordered = sorted(values)
index = int((len(ordered) - 1) * fraction)
return round(ordered[index], 2)
def main():
def configure(parser):
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--duration", type=int, default=30)
parser.add_argument(
"--output",
default="results/upload-memory.json",
)
args = arguments(
"Concurrent image uploads and API memory measurement",
configure,
)
check(
args.concurrency >= 10,
"Use at least 10 concurrent clients",
)
check(
args.duration >= 10,
"Duration must be at least 10 seconds",
)
controller = Client(
args.base_url,
args.ca_file,
timeout=40,
)
alice, bob, eve, chat = controller.bootstrap()
containers = get_api_containers()
print("Generating PNG 6000x6000...", flush=True)
image = make_png(6000, 6000)
checksum = hashlib.sha256(image).hexdigest()
check(
len(image) <= 10 * 1024 * 1024,
"Generated image exceeds upload byte limit",
)
print(
f"Image: {len(image)} bytes, 36000000 pixels",
flush=True,
)
path = (
"/api/v1/chats/"
+ chat["id"]
+ "/attachments?filename=memory-test.png"
)
clients = [
Client(args.base_url, args.ca_file, timeout=40)
for _ in range(args.concurrency)
]
baseline = {
container: memory_snapshot(container)
for container in containers
}
samples = {container: [] for container in containers}
monitor_errors = []
stop_monitor = threading.Event()
started = time.monotonic()
def monitor():
while not stop_monitor.is_set():
for container in containers:
try:
snapshot = memory_snapshot(container)
samples[container].append({
"elapsed_seconds": round(
time.monotonic() - started,
3,
),
**snapshot,
})
except Exception as error:
monitor_errors.append(
type(error).__name__
)
stop_monitor.set()
return
stop_monitor.wait(0.5)
barrier = threading.Barrier(args.concurrency + 1)
def worker(index):
client = clients[index]
counters = Counter()
latencies = []
errors = []
instances = set()
successful_instances = set()
first_success = None
barrier.wait()
while time.monotonic() - started < args.duration:
request_started = time.monotonic()
try:
response = client.call(
"POST",
path,
expected=[201, 429],
actor=alice,
raw=image,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
instance = response["headers"]["X-Instance-Id"]
instances.add(instance)
if response["status"] == 429:
check(
response["json"]["error"]["code"]
== "rate_limited",
"Unexpected error code for 429",
)
counters["rate_limited"] += 1
else:
attachment = response["json"]
check(
attachment["status"] == "ready",
"Successful upload is not ready",
)
check(
attachment["sha256"] == checksum,
"Uploaded SHA-256 mismatch",
)
check(
attachment["size_bytes"] == len(image),
"Uploaded size mismatch",
)
counters["successful"] += 1
successful_instances.add(instance)
latencies.append(
(time.monotonic() - request_started) * 1000
)
if first_success is None:
first_success = attachment
except Exception as error:
counters["unexpected_errors"] += 1
if len(errors) < 3:
errors.append(str(error))
time.sleep(0.1)
return {
"counters": dict(counters),
"latencies": latencies,
"errors": errors,
"instances": instances,
"successful_instances": successful_instances,
"first_success": first_success,
}
monitor_thread = threading.Thread(
target=monitor,
daemon=True,
)
with ThreadPoolExecutor(
max_workers=args.concurrency
) as executor:
futures = [
executor.submit(worker, index)
for index in range(args.concurrency)
]
started = time.monotonic()
monitor_thread.start()
print(
f"Starting {args.concurrency} clients "
f"for {args.duration} seconds...",
flush=True,
)
barrier.wait()
worker_results = [
future.result()
for future in futures
]
elapsed = time.monotonic() - started
stop_monitor.set()
monitor_thread.join(timeout=25)
check(
not monitor_thread.is_alive(),
"Memory monitor did not stop",
)
final = {
container: memory_snapshot(container)
for container in containers
}
totals = Counter()
latencies = []
errors = []
instances = set()
successful_instances = set()
for result in worker_results:
totals.update(result["counters"])
latencies.extend(result["latencies"])
instances.update(result["instances"])
successful_instances.update(
result["successful_instances"]
)
errors.extend(result["errors"])
download_errors = []
for result in worker_results:
attachment = result["first_success"]
if attachment is None:
continue
try:
response = controller.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=bob,
)
check(
response["bytes"] == image,
"Downloaded image differs from the original",
)
except Exception as error:
download_errors.append(str(error))
memory_report = {}
for container in containers:
measurements = [
baseline[container]["current_bytes"],
final[container]["current_bytes"],
]
measurements.extend(
sample["current_bytes"]
for sample in samples[container]
)
memory_report[container[:12]] = {
"baseline_mib": mib(
baseline[container]["current_bytes"]
),
"observed_peak_mib": mib(max(measurements)),
"final_mib": mib(
final[container]["current_bytes"]
),
"lifetime_peak_before_mib": mib(
baseline[container]["lifetime_peak_bytes"]
),
"lifetime_peak_after_mib": mib(
final[container]["lifetime_peak_bytes"]
),
"oom_delta": (
final[container]["oom"]
- baseline[container]["oom"]
),
"oom_kill_delta": (
final[container]["oom_kill"]
- baseline[container]["oom_kill"]
),
"sample_count": len(samples[container]),
}
success = totals["successful"]
limited = totals["rate_limited"]
unexpected = totals["unexpected_errors"]
passed = (
success > 0
and unexpected == 0
and not monitor_errors
and not download_errors
and len(successful_instances) >= 2
and all(
item["sample_count"] > 0
and item["oom_delta"] == 0
and item["oom_kill_delta"] == 0
for item in memory_report.values()
)
)
report = {
"passed": passed,
"workload": "closed-loop image uploads; no automatic retries",
"concurrency": args.concurrency,
"requested_duration_seconds": args.duration,
"elapsed_seconds": round(elapsed, 3),
"image": {
"width": 6000,
"height": 6000,
"pixels": 36_000_000,
"size_bytes": len(image),
"sha256": checksum,
},
"successful_uploads": success,
"rate_limited_requests": limited,
"unexpected_errors": unexpected,
"successful_rps": round(success / elapsed, 3),
"observed_instances": sorted(instances),
"successful_instances": sorted(successful_instances),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"memory": memory_report,
"error_examples": errors[:10],
"monitor_errors": monitor_errors,
"download_errors": download_errors,
"notes": [
"429 responses are admission-control rejections, not successful uploads.",
"Observed memory peak is sampled and may miss short spikes.",
"Lifetime cgroup peak includes earlier container activity and startup.",
"Container memory includes more than Java heap.",
"Concurrency means concurrent clients, not 10 simultaneous decoders.",
],
}
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
saved_report = {
**report,
"memory_samples": samples,
}
output.write_text(
json.dumps(saved_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
print(
json.dumps(report, ensure_ascii=False, indent=2),
flush=True,
)
print("Report saved to:", output, flush=True)
check(
passed,
"Experiment detected errors; inspect the saved report",
)
print("PASS: concurrent upload experiment completed", flush=True)
if __name__ == "__main__":
run(main)