forked from mtusi2026/mtusi2026lab4
all for lab4
This commit is contained in:
@@ -0,0 +1,495 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from attachments_negative import make_png
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def docker_output(command):
|
||||
result = subprocess.run(
|
||||
command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
check(
|
||||
result.returncode == 0,
|
||||
"Docker command failed while measuring memory",
|
||||
)
|
||||
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def get_api_containers():
|
||||
output = docker_output([
|
||||
"docker", "compose", "ps", "-q", "api",
|
||||
])
|
||||
|
||||
containers = output.splitlines()
|
||||
|
||||
check(
|
||||
len(containers) == 2,
|
||||
"Expected exactly two running API replicas",
|
||||
)
|
||||
|
||||
return containers
|
||||
|
||||
|
||||
def memory_snapshot(container):
|
||||
# Docker Desktop с cgroup v2.
|
||||
# memory.current включает память всего контейнера,
|
||||
# а не только Java heap.
|
||||
output = docker_output([
|
||||
"docker", "exec", container,
|
||||
"sh", "-ec",
|
||||
'''
|
||||
cat /sys/fs/cgroup/memory.current
|
||||
|
||||
if [ -r /sys/fs/cgroup/memory.peak ]; then
|
||||
cat /sys/fs/cgroup/memory.peak
|
||||
else
|
||||
echo unavailable
|
||||
fi
|
||||
|
||||
cat /sys/fs/cgroup/memory.events
|
||||
''',
|
||||
])
|
||||
|
||||
lines = output.splitlines()
|
||||
|
||||
check(
|
||||
len(lines) >= 2,
|
||||
"Cannot read cgroup v2 memory statistics",
|
||||
)
|
||||
|
||||
events = {}
|
||||
|
||||
for line in lines[2:]:
|
||||
key, value = line.split()
|
||||
events[key] = int(value)
|
||||
|
||||
return {
|
||||
"current_bytes": int(lines[0]),
|
||||
"lifetime_peak_bytes": (
|
||||
int(lines[1])
|
||||
if lines[1].isdigit()
|
||||
else None
|
||||
),
|
||||
"oom": events.get("oom", 0),
|
||||
"oom_kill": events.get("oom_kill", 0),
|
||||
}
|
||||
|
||||
|
||||
def mib(value):
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
return round(value / 1024 / 1024, 2)
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
if not values:
|
||||
return None
|
||||
|
||||
ordered = sorted(values)
|
||||
index = int((len(ordered) - 1) * fraction)
|
||||
|
||||
return round(ordered[index], 2)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(parser):
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--duration", type=int, default=30)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="results/upload-memory.json",
|
||||
)
|
||||
|
||||
args = arguments(
|
||||
"Concurrent image uploads and API memory measurement",
|
||||
configure,
|
||||
)
|
||||
|
||||
check(
|
||||
args.concurrency >= 10,
|
||||
"Use at least 10 concurrent clients",
|
||||
)
|
||||
|
||||
check(
|
||||
args.duration >= 10,
|
||||
"Duration must be at least 10 seconds",
|
||||
)
|
||||
|
||||
controller = Client(
|
||||
args.base_url,
|
||||
args.ca_file,
|
||||
timeout=40,
|
||||
)
|
||||
|
||||
alice, bob, eve, chat = controller.bootstrap()
|
||||
containers = get_api_containers()
|
||||
|
||||
print("Generating PNG 6000x6000...", flush=True)
|
||||
|
||||
image = make_png(6000, 6000)
|
||||
checksum = hashlib.sha256(image).hexdigest()
|
||||
|
||||
check(
|
||||
len(image) <= 10 * 1024 * 1024,
|
||||
"Generated image exceeds upload byte limit",
|
||||
)
|
||||
|
||||
print(
|
||||
f"Image: {len(image)} bytes, 36000000 pixels",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
path = (
|
||||
"/api/v1/chats/"
|
||||
+ chat["id"]
|
||||
+ "/attachments?filename=memory-test.png"
|
||||
)
|
||||
|
||||
clients = [
|
||||
Client(args.base_url, args.ca_file, timeout=40)
|
||||
for _ in range(args.concurrency)
|
||||
]
|
||||
|
||||
baseline = {
|
||||
container: memory_snapshot(container)
|
||||
for container in containers
|
||||
}
|
||||
|
||||
samples = {container: [] for container in containers}
|
||||
monitor_errors = []
|
||||
stop_monitor = threading.Event()
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
def monitor():
|
||||
while not stop_monitor.is_set():
|
||||
for container in containers:
|
||||
try:
|
||||
snapshot = memory_snapshot(container)
|
||||
|
||||
samples[container].append({
|
||||
"elapsed_seconds": round(
|
||||
time.monotonic() - started,
|
||||
3,
|
||||
),
|
||||
**snapshot,
|
||||
})
|
||||
|
||||
except Exception as error:
|
||||
monitor_errors.append(
|
||||
type(error).__name__
|
||||
)
|
||||
stop_monitor.set()
|
||||
return
|
||||
|
||||
stop_monitor.wait(0.5)
|
||||
|
||||
barrier = threading.Barrier(args.concurrency + 1)
|
||||
|
||||
def worker(index):
|
||||
client = clients[index]
|
||||
|
||||
counters = Counter()
|
||||
latencies = []
|
||||
errors = []
|
||||
instances = set()
|
||||
successful_instances = set()
|
||||
first_success = None
|
||||
|
||||
barrier.wait()
|
||||
|
||||
while time.monotonic() - started < args.duration:
|
||||
request_started = time.monotonic()
|
||||
|
||||
try:
|
||||
response = client.call(
|
||||
"POST",
|
||||
path,
|
||||
expected=[201, 429],
|
||||
actor=alice,
|
||||
raw=image,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
|
||||
instance = response["headers"]["X-Instance-Id"]
|
||||
instances.add(instance)
|
||||
|
||||
if response["status"] == 429:
|
||||
check(
|
||||
response["json"]["error"]["code"]
|
||||
== "rate_limited",
|
||||
"Unexpected error code for 429",
|
||||
)
|
||||
|
||||
counters["rate_limited"] += 1
|
||||
|
||||
else:
|
||||
attachment = response["json"]
|
||||
|
||||
check(
|
||||
attachment["status"] == "ready",
|
||||
"Successful upload is not ready",
|
||||
)
|
||||
|
||||
check(
|
||||
attachment["sha256"] == checksum,
|
||||
"Uploaded SHA-256 mismatch",
|
||||
)
|
||||
|
||||
check(
|
||||
attachment["size_bytes"] == len(image),
|
||||
"Uploaded size mismatch",
|
||||
)
|
||||
|
||||
counters["successful"] += 1
|
||||
successful_instances.add(instance)
|
||||
|
||||
latencies.append(
|
||||
(time.monotonic() - request_started) * 1000
|
||||
)
|
||||
|
||||
if first_success is None:
|
||||
first_success = attachment
|
||||
|
||||
except Exception as error:
|
||||
counters["unexpected_errors"] += 1
|
||||
|
||||
if len(errors) < 3:
|
||||
errors.append(str(error))
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
return {
|
||||
"counters": dict(counters),
|
||||
"latencies": latencies,
|
||||
"errors": errors,
|
||||
"instances": instances,
|
||||
"successful_instances": successful_instances,
|
||||
"first_success": first_success,
|
||||
}
|
||||
|
||||
monitor_thread = threading.Thread(
|
||||
target=monitor,
|
||||
daemon=True,
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=args.concurrency
|
||||
) as executor:
|
||||
|
||||
futures = [
|
||||
executor.submit(worker, index)
|
||||
for index in range(args.concurrency)
|
||||
]
|
||||
|
||||
started = time.monotonic()
|
||||
monitor_thread.start()
|
||||
|
||||
print(
|
||||
f"Starting {args.concurrency} clients "
|
||||
f"for {args.duration} seconds...",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
barrier.wait()
|
||||
|
||||
worker_results = [
|
||||
future.result()
|
||||
for future in futures
|
||||
]
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
|
||||
stop_monitor.set()
|
||||
monitor_thread.join(timeout=25)
|
||||
|
||||
check(
|
||||
not monitor_thread.is_alive(),
|
||||
"Memory monitor did not stop",
|
||||
)
|
||||
|
||||
final = {
|
||||
container: memory_snapshot(container)
|
||||
for container in containers
|
||||
}
|
||||
|
||||
totals = Counter()
|
||||
latencies = []
|
||||
errors = []
|
||||
instances = set()
|
||||
successful_instances = set()
|
||||
|
||||
for result in worker_results:
|
||||
totals.update(result["counters"])
|
||||
latencies.extend(result["latencies"])
|
||||
instances.update(result["instances"])
|
||||
successful_instances.update(
|
||||
result["successful_instances"]
|
||||
)
|
||||
|
||||
errors.extend(result["errors"])
|
||||
|
||||
download_errors = []
|
||||
|
||||
for result in worker_results:
|
||||
attachment = result["first_success"]
|
||||
|
||||
if attachment is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
response = controller.call(
|
||||
"GET",
|
||||
"/api/v1/attachments/"
|
||||
+ attachment["id"]
|
||||
+ "/content",
|
||||
actor=bob,
|
||||
)
|
||||
|
||||
check(
|
||||
response["bytes"] == image,
|
||||
"Downloaded image differs from the original",
|
||||
)
|
||||
|
||||
except Exception as error:
|
||||
download_errors.append(str(error))
|
||||
|
||||
memory_report = {}
|
||||
|
||||
for container in containers:
|
||||
measurements = [
|
||||
baseline[container]["current_bytes"],
|
||||
final[container]["current_bytes"],
|
||||
]
|
||||
|
||||
measurements.extend(
|
||||
sample["current_bytes"]
|
||||
for sample in samples[container]
|
||||
)
|
||||
|
||||
memory_report[container[:12]] = {
|
||||
"baseline_mib": mib(
|
||||
baseline[container]["current_bytes"]
|
||||
),
|
||||
"observed_peak_mib": mib(max(measurements)),
|
||||
"final_mib": mib(
|
||||
final[container]["current_bytes"]
|
||||
),
|
||||
"lifetime_peak_before_mib": mib(
|
||||
baseline[container]["lifetime_peak_bytes"]
|
||||
),
|
||||
"lifetime_peak_after_mib": mib(
|
||||
final[container]["lifetime_peak_bytes"]
|
||||
),
|
||||
"oom_delta": (
|
||||
final[container]["oom"]
|
||||
- baseline[container]["oom"]
|
||||
),
|
||||
"oom_kill_delta": (
|
||||
final[container]["oom_kill"]
|
||||
- baseline[container]["oom_kill"]
|
||||
),
|
||||
"sample_count": len(samples[container]),
|
||||
}
|
||||
|
||||
success = totals["successful"]
|
||||
limited = totals["rate_limited"]
|
||||
unexpected = totals["unexpected_errors"]
|
||||
|
||||
passed = (
|
||||
success > 0
|
||||
and unexpected == 0
|
||||
and not monitor_errors
|
||||
and not download_errors
|
||||
and len(successful_instances) >= 2
|
||||
and all(
|
||||
item["sample_count"] > 0
|
||||
and item["oom_delta"] == 0
|
||||
and item["oom_kill_delta"] == 0
|
||||
for item in memory_report.values()
|
||||
)
|
||||
)
|
||||
|
||||
report = {
|
||||
"passed": passed,
|
||||
"workload": "closed-loop image uploads; no automatic retries",
|
||||
"concurrency": args.concurrency,
|
||||
"requested_duration_seconds": args.duration,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"image": {
|
||||
"width": 6000,
|
||||
"height": 6000,
|
||||
"pixels": 36_000_000,
|
||||
"size_bytes": len(image),
|
||||
"sha256": checksum,
|
||||
},
|
||||
"successful_uploads": success,
|
||||
"rate_limited_requests": limited,
|
||||
"unexpected_errors": unexpected,
|
||||
"successful_rps": round(success / elapsed, 3),
|
||||
"observed_instances": sorted(instances),
|
||||
"successful_instances": sorted(successful_instances),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"memory": memory_report,
|
||||
"error_examples": errors[:10],
|
||||
"monitor_errors": monitor_errors,
|
||||
"download_errors": download_errors,
|
||||
"notes": [
|
||||
"429 responses are admission-control rejections, not successful uploads.",
|
||||
"Observed memory peak is sampled and may miss short spikes.",
|
||||
"Lifetime cgroup peak includes earlier container activity and startup.",
|
||||
"Container memory includes more than Java heap.",
|
||||
"Concurrency means concurrent clients, not 10 simultaneous decoders.",
|
||||
],
|
||||
}
|
||||
|
||||
output = Path(args.output)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
saved_report = {
|
||||
**report,
|
||||
"memory_samples": samples,
|
||||
}
|
||||
|
||||
output.write_text(
|
||||
json.dumps(saved_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
print(
|
||||
json.dumps(report, ensure_ascii=False, indent=2),
|
||||
flush=True,
|
||||
)
|
||||
|
||||
print("Report saved to:", output, flush=True)
|
||||
|
||||
check(
|
||||
passed,
|
||||
"Experiment detected errors; inspect the saved report",
|
||||
)
|
||||
|
||||
print("PASS: concurrent upload experiment completed", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Reference in New Issue
Block a user