Files
mtusi2026lab5/scripts/measure-upload-memory.py
T

175 lines
5.6 KiB
Python
Executable File

#!/usr/bin/env python3
"""Measure API container memory while concurrent uploads run.
The HTTP workload runs in a throwaway Python container attached to the Compose
network. This host-side script samples the Docker Engine stats API at a short
interval, because the repository shell cannot always reach published ports and
`docker stats --no-stream` is too slow to catch a short upload burst.
"""
import argparse
import json
import socket
import subprocess
import sys
import threading
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
DOCKER_SOCKET = "/var/run/docker.sock"
def docker_json(path):
request = f"GET {path} HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"
with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock:
sock.connect(DOCKER_SOCKET)
sock.sendall(request.encode())
data = bytearray()
while True:
part = sock.recv(65536)
if not part:
break
data.extend(part)
head, _, body = bytes(data).partition(b"\r\n\r\n")
if b"transfer-encoding: chunked" not in head.lower():
return json.loads(body)
# The Engine API uses chunked transfer encoding for stats and inspect.
decoded = bytearray()
position = 0
while position < len(body):
line_end = body.find(b"\r\n", position)
if line_end < 0:
break
size = int(body[position:line_end].split(b";", 1)[0], 16)
if size == 0:
break
start = line_end + 2
decoded.extend(body[start : start + size])
position = start + size + 2
return json.loads(bytes(decoded))
def compose(*args):
return subprocess.check_output(
["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True
).strip()
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--uploads", type=int, default=10)
parser.add_argument("--width", type=int, default=1800)
parser.add_argument("--height", type=int, default=1400)
parser.add_argument("--interval", type=float, default=0.05)
parser.add_argument("--output", type=Path, default=ROOT / "evidence/upload-memory.json")
args = parser.parse_args()
api_ids = [line for line in compose("ps", "-q", "api").splitlines() if line]
if len(api_ids) < 2:
raise SystemExit("at least two running API replicas are required")
names = {
container_id: docker_json(f"/containers/{container_id}/json")["Name"].lstrip("/")
for container_id in api_ids
}
network = docker_json(
f"/containers/{compose('ps', '-q', 'haproxy')}/json"
)["NetworkSettings"]["Networks"]
network_name = next(iter(network))
stop = threading.Event()
samples = []
def sample():
while not stop.is_set():
per_container = {}
for container_id in api_ids:
stats = docker_json(
f"/containers/{container_id}/stats?stream=false&one-shot=true"
)
per_container[names[container_id]] = stats.get("memory_stats", {}).get(
"usage", 0
)
samples.append(
{
"at": time.time(),
"total": sum(per_container.values()),
"per_container": per_container,
}
)
time.sleep(args.interval)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
time.sleep(0.3) # capture an idle baseline before the burst
command = [
"docker",
"run",
"--rm",
"--network",
network_name,
"-v",
f"{ROOT}:/work:ro",
"-w",
"/work",
"python:3.12-alpine",
"python3",
"tests/custom/concurrent_uploads.py",
"--base-url",
"http://haproxy:8080",
"--uploads",
str(args.uploads),
"--width",
str(args.width),
"--height",
str(args.height),
]
started = time.monotonic()
try:
output = subprocess.check_output(command, text=True)
finally:
stop.set()
sampler.join(timeout=2)
workload = json.loads(output.strip().splitlines()[-1])
baseline = samples[0]["total"] if samples else 0
peak = max(samples, key=lambda item: item["total"]) if samples else {"total": 0}
report = {
"workload": workload,
"sampling_interval_seconds": args.interval,
"samples": len(samples),
"wall_seconds": round(time.monotonic() - started, 3),
"baseline_total_mib": round(baseline / (1024 * 1024), 1),
"peak_total_mib": round(peak["total"] / (1024 * 1024), 1),
"peak_per_container_mib": {
name: round(value / (1024 * 1024), 1)
for name, value in peak["per_container"].items()
},
"api_memory_limit_mib": None,
}
# The limit is configured in .env; keep the evidence honest by reading the
# effective value from Compose rather than repeating a default.
configured = subprocess.check_output(
[
"docker",
"inspect",
"-f",
"{{.HostConfig.Memory}}",
api_ids[0],
],
text=True,
).strip()
limit_bytes = int(configured or 0)
report["api_memory_limit_mib"] = (
round(limit_bytes / (1024 * 1024), 1) if limit_bytes else None
)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n")
print(json.dumps(report, indent=2))
if __name__ == "__main__":
main()