130 lines
6.5 KiB
Python
130 lines
6.5 KiB
Python
from concurrent.futures import ThreadPoolExecutor
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlencode
|
|
from uuid import uuid4
|
|
|
|
from helpers import Client, ROOT, image_bytes, lab, padded_png
|
|
|
|
|
|
RUNTIME = ("lb", "api1", "api2", "cleanup", "s3", "s3-storage", "db")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--project", required=True)
|
|
parser.add_argument("--port", required=True, type=int)
|
|
args = parser.parse_args()
|
|
args.ttl, args.test_port = 3600, args.port
|
|
samplers, records, readers = {}, {"api1": [], "api2": []}, []
|
|
result = {"clients": 10, "upload_slots_per_replica": 2, "sampling_seconds": 0.05, "profiles": []}
|
|
flag = "/tmp/memory-stop-" + uuid4().hex
|
|
code = (
|
|
"import json,time; from pathlib import Path\n"
|
|
"root=Path('/sys/fs/cgroup')\n"
|
|
"while not Path(" + repr(flag) + ").exists():\n"
|
|
" status=dict(line.split(':',1) for line in Path('/proc/1/status').read_text().splitlines())\n"
|
|
" events=dict(line.split() for line in (root/'memory.events').read_text().splitlines())\n"
|
|
" print(json.dumps({'current':int((root/'memory.current').read_text()),"
|
|
"'peak':int((root/'memory.peak').read_text()),'limit':int((root/'memory.max').read_text()),"
|
|
"'rss':int(status['VmRSS'].split()[0])*1024,'hwm':int(status['VmHWM'].split()[0])*1024,"
|
|
"'oom_kill':int(events['oom_kill'])}),flush=True)\n"
|
|
" time.sleep(0.05)"
|
|
)
|
|
try:
|
|
# свежие API дают отдельный cgroup peak для этого опыта
|
|
lab.compose(args, "stop", "--timeout", "3", *RUNTIME)
|
|
lab.compose(args, "rm", "-f", "api1", "api2")
|
|
lab.start(args)
|
|
client = Client(lab.base_url(args), timeout=60)
|
|
actor, bob, eve, chat = client.bootstrap()
|
|
initial = lab.states(args)
|
|
for service in records:
|
|
command = ["docker", "exec", initial[service]["id"], "python", "-u", "-c", code]
|
|
process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
|
|
text=True, encoding="utf-8")
|
|
samplers[service] = process
|
|
def collect(name=service, running=process):
|
|
for line in running.stdout:
|
|
item = json.loads(line)
|
|
item["received_at"] = time.monotonic()
|
|
records[name].append(item)
|
|
reader = threading.Thread(target=collect)
|
|
reader.start()
|
|
readers.append(reader)
|
|
deadline = time.monotonic() + 10
|
|
while not all(records.values()):
|
|
if time.monotonic() > deadline:
|
|
raise AssertionError("Memory sampler did not start")
|
|
time.sleep(0.05)
|
|
result["baseline"] = {name: values[-1] for name, values in records.items()}
|
|
profiles = [
|
|
("near_byte_limit", padded_png(10 * 1024 * 1024 - 1024), [32, 24]),
|
|
("near_pixel_limit", image_bytes(size=(8000, 5000), mode="RGBA"), [8000, 5000]),
|
|
]
|
|
for name, raw, dimensions in profiles:
|
|
started = time.monotonic()
|
|
barrier = threading.Barrier(10)
|
|
def upload(_):
|
|
c = Client(client.base, timeout=60)
|
|
barrier.wait(timeout=15)
|
|
response = c.call(
|
|
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?"
|
|
+ urlencode({"filename": name + ".png"}), 201, actor=actor, raw=raw,
|
|
headers={"Content-Type": "image/png"}, schema="Attachment",
|
|
)
|
|
return response["json"], response["headers"]["X-Instance-Id"], response["elapsed"]
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
uploads = list(pool.map(upload, range(10)))
|
|
for att, instance, elapsed in uploads:
|
|
downloaded = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)["bytes"]
|
|
assert hashlib.sha256(downloaded).hexdigest() == att["sha256"] == hashlib.sha256(raw).hexdigest()
|
|
assert downloaded == raw
|
|
peaks = {}
|
|
for service, values in records.items():
|
|
window = [item for item in values if item["received_at"] >= started]
|
|
assert window, "No memory samples during upload"
|
|
peaks[service] = {
|
|
"sampled_cgroup_peak_bytes": max(x["current"] for x in window),
|
|
"sampled_rss_peak_bytes": max(x["rss"] for x in window),
|
|
"cgroup_lifetime_peak_bytes": window[-1]["peak"],
|
|
"process_lifetime_hwm_bytes": window[-1]["hwm"],
|
|
"memory_limit_bytes": window[-1]["limit"],
|
|
"oom_kills": window[-1]["oom_kill"],
|
|
}
|
|
assert peaks[service]["oom_kills"] == 0
|
|
assert peaks[service]["cgroup_lifetime_peak_bytes"] < peaks[service]["memory_limit_bytes"]
|
|
assert len({item[0]["id"] for item in uploads}) == 10
|
|
assert len({item[1] for item in uploads}) == 2
|
|
result["profiles"].append({
|
|
"name": name, "file_bytes": len(raw), "dimensions": dimensions, "successes": len(uploads),
|
|
"errors": 0, "seconds": round(time.monotonic() - started, 3),
|
|
"slowest_upload_seconds": round(max(item[2] for item in uploads), 3),
|
|
"replicas_used": 2, "originals_verified": 10, "memory": peaks,
|
|
})
|
|
for service in records:
|
|
data = json.loads(lab.run(args, ["docker", "inspect", "--format",
|
|
'{"oom":{{json .State.OOMKilled}},"restarts":{{.RestartCount}}}',
|
|
initial[service]["id"]], quiet=True))
|
|
assert not data["oom"] and data["restarts"] == 0
|
|
result["status"] = "PASS"
|
|
lab.evidence("memory.json", result)
|
|
print("PASS: 10 concurrent uploads in both profiles; originals intact, no OOM or restarts")
|
|
finally:
|
|
for service, process in samplers.items():
|
|
lab.compose(args, "exec", "-T", service, "python", "-c", "from pathlib import Path; Path(" + repr(flag) + ").touch()",
|
|
quiet=True)
|
|
process.wait(timeout=10)
|
|
for reader in readers:
|
|
reader.join(timeout=5)
|
|
lab.compose(args, "stop", "--timeout", "10", *RUNTIME)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|