162 lines
5.7 KiB
Python
162 lines
5.7 KiB
Python
"""Ten simultaneous HTTP uploads through LB, with kernel memory high-water readings."""
|
|
|
|
import argparse
|
|
import json
|
|
import random
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
from PIL import Image
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
from client import Client
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--output", default="evidence/upload-memory.json")
|
|
parser.add_argument("--base-url", default="http://127.0.0.1:8084")
|
|
parser.add_argument("--restart-api", action="store_true")
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url)
|
|
alice, _, _, chat = c.bootstrap()
|
|
if args.restart_api:
|
|
subprocess.run(
|
|
["docker", "compose", "restart", "api"], check=True, stdout=subprocess.DEVNULL
|
|
)
|
|
for _ in range(40):
|
|
try:
|
|
c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
|
|
break
|
|
except AssertionError:
|
|
time.sleep(0.5)
|
|
else:
|
|
raise AssertionError("API did not recover with old token")
|
|
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
|
|
|
|
api_count = len(ids)
|
|
ids += subprocess.check_output(["docker", "compose", "ps", "-q", "lb"], text=True).split()
|
|
|
|
def memory():
|
|
result = {}
|
|
for container in ids:
|
|
lines = subprocess.check_output(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
container,
|
|
"cat",
|
|
"/sys/fs/cgroup/memory.current",
|
|
"/sys/fs/cgroup/memory.peak",
|
|
"/sys/fs/cgroup/memory.events",
|
|
"/proc/1/status",
|
|
],
|
|
text=True,
|
|
).splitlines()
|
|
result[container] = {
|
|
"current": int(lines[0]),
|
|
"peak": int(lines[1]),
|
|
"events": {
|
|
line.split()[0]: int(line.split()[1])
|
|
for line in lines[2:]
|
|
if line.split()
|
|
and line.split()[0]
|
|
in {"low", "high", "max", "oom", "oom_kill", "oom_group_kill"}
|
|
},
|
|
"process_hwm": next(line for line in lines if line.startswith("VmHWM:")),
|
|
}
|
|
return result
|
|
|
|
result = {
|
|
"concurrency": 10,
|
|
"api_replicas": api_count,
|
|
"sampled_api_and_lb_containers": ids,
|
|
"api_restarted_before_trial": args.restart_api,
|
|
"runs": [],
|
|
"limitations": [
|
|
"Kernel cgroup peak and process VmHWM are lifetime peaks; before/after reported, not attributed solely to upload buffers",
|
|
"Fast rejection 429 is intentional backpressure, not a successful upload",
|
|
"Local two-replica stand; not a throughput benchmark",
|
|
],
|
|
}
|
|
for side in (512, 1700):
|
|
output = BytesIO()
|
|
Image.frombytes("RGB", (side, side), random.Random(42).randbytes(side * side * 3)).save(
|
|
output, "PNG"
|
|
)
|
|
data = output.getvalue()
|
|
barrier = threading.Barrier(10)
|
|
before = memory()
|
|
|
|
def send(_, barrier=barrier, data=data):
|
|
barrier.wait()
|
|
started = time.perf_counter()
|
|
try:
|
|
with httpx.Client(timeout=20, trust_env=False) as client:
|
|
response = client.post(
|
|
args.base_url
|
|
+ f"/api/v1/chats/{chat['id']}/attachments?filename=memory.png",
|
|
content=data,
|
|
headers={
|
|
"Authorization": "Bearer " + alice["token"],
|
|
"Content-Type": "image/png",
|
|
},
|
|
)
|
|
record = {
|
|
"status": response.status_code,
|
|
"elapsed": time.perf_counter() - started,
|
|
}
|
|
if response.status_code == 201:
|
|
record["attachment_id"] = response.json()["id"]
|
|
return record
|
|
except httpx.HTTPError as exc:
|
|
return {"error": type(exc).__name__, "elapsed": time.perf_counter() - started}
|
|
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
responses = list(pool.map(send, range(10)))
|
|
after = memory()
|
|
c.call("GET", "/health/ready", schema="Health")
|
|
for record in responses:
|
|
if record.get("status") not in {201, 429}:
|
|
raise AssertionError(record)
|
|
for container in ids:
|
|
state = json.loads(
|
|
subprocess.check_output(["docker", "inspect", container], text=True)
|
|
)[0]["State"]
|
|
if state["OOMKilled"] or not state["Running"]:
|
|
raise AssertionError("API OOM or not running")
|
|
result["runs"].append(
|
|
{
|
|
"image_dimensions": [side, side],
|
|
"size_bytes": len(data),
|
|
"before": before,
|
|
"after": after,
|
|
"responses": responses,
|
|
"ready_after": True,
|
|
}
|
|
)
|
|
Path(args.output).write_text(json.dumps(result, indent=2) + "\n")
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"runs": [
|
|
{
|
|
"size_bytes": r["size_bytes"],
|
|
"statuses": [x.get("status", x.get("error")) for x in r["responses"]],
|
|
}
|
|
for r in result["runs"]
|
|
]
|
|
}
|
|
)
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|