Files
mtusi2026lab4/student-tests/check_memory.py
T

130 lines
6.5 KiB
Python

from concurrent.futures import ThreadPoolExecutor
import argparse
import hashlib
import json
from pathlib import Path
import subprocess
import threading
import time
from urllib.parse import urlencode
from uuid import uuid4
from helpers import Client, ROOT, image_bytes, lab, padded_png
RUNTIME = ("lb", "api1", "api2", "cleanup", "s3", "s3-storage", "db")
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--project", required=True)
parser.add_argument("--port", required=True, type=int)
args = parser.parse_args()
args.ttl, args.test_port = 3600, args.port
samplers, records, readers = {}, {"api1": [], "api2": []}, []
result = {"clients": 10, "upload_slots_per_replica": 2, "sampling_seconds": 0.05, "profiles": []}
flag = "/tmp/memory-stop-" + uuid4().hex
code = (
"import json,time; from pathlib import Path\n"
"root=Path('/sys/fs/cgroup')\n"
"while not Path(" + repr(flag) + ").exists():\n"
" status=dict(line.split(':',1) for line in Path('/proc/1/status').read_text().splitlines())\n"
" events=dict(line.split() for line in (root/'memory.events').read_text().splitlines())\n"
" print(json.dumps({'current':int((root/'memory.current').read_text()),"
"'peak':int((root/'memory.peak').read_text()),'limit':int((root/'memory.max').read_text()),"
"'rss':int(status['VmRSS'].split()[0])*1024,'hwm':int(status['VmHWM'].split()[0])*1024,"
"'oom_kill':int(events['oom_kill'])}),flush=True)\n"
" time.sleep(0.05)"
)
try:
# свежие API дают отдельный cgroup peak для этого опыта
lab.compose(args, "stop", "--timeout", "3", *RUNTIME)
lab.compose(args, "rm", "-f", "api1", "api2")
lab.start(args)
client = Client(lab.base_url(args), timeout=60)
actor, bob, eve, chat = client.bootstrap()
initial = lab.states(args)
for service in records:
command = ["docker", "exec", initial[service]["id"], "python", "-u", "-c", code]
process = subprocess.Popen(command, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, encoding="utf-8")
samplers[service] = process
def collect(name=service, running=process):
for line in running.stdout:
item = json.loads(line)
item["received_at"] = time.monotonic()
records[name].append(item)
reader = threading.Thread(target=collect)
reader.start()
readers.append(reader)
deadline = time.monotonic() + 10
while not all(records.values()):
if time.monotonic() > deadline:
raise AssertionError("Memory sampler did not start")
time.sleep(0.05)
result["baseline"] = {name: values[-1] for name, values in records.items()}
profiles = [
("near_byte_limit", padded_png(10 * 1024 * 1024 - 1024), [32, 24]),
("near_pixel_limit", image_bytes(size=(8000, 5000), mode="RGBA"), [8000, 5000]),
]
for name, raw, dimensions in profiles:
started = time.monotonic()
barrier = threading.Barrier(10)
def upload(_):
c = Client(client.base, timeout=60)
barrier.wait(timeout=15)
response = c.call(
"POST", "/api/v1/chats/" + chat["id"] + "/attachments?"
+ urlencode({"filename": name + ".png"}), 201, actor=actor, raw=raw,
headers={"Content-Type": "image/png"}, schema="Attachment",
)
return response["json"], response["headers"]["X-Instance-Id"], response["elapsed"]
with ThreadPoolExecutor(max_workers=10) as pool:
uploads = list(pool.map(upload, range(10)))
for att, instance, elapsed in uploads:
downloaded = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)["bytes"]
assert hashlib.sha256(downloaded).hexdigest() == att["sha256"] == hashlib.sha256(raw).hexdigest()
assert downloaded == raw
peaks = {}
for service, values in records.items():
window = [item for item in values if item["received_at"] >= started]
assert window, "No memory samples during upload"
peaks[service] = {
"sampled_cgroup_peak_bytes": max(x["current"] for x in window),
"sampled_rss_peak_bytes": max(x["rss"] for x in window),
"cgroup_lifetime_peak_bytes": window[-1]["peak"],
"process_lifetime_hwm_bytes": window[-1]["hwm"],
"memory_limit_bytes": window[-1]["limit"],
"oom_kills": window[-1]["oom_kill"],
}
assert peaks[service]["oom_kills"] == 0
assert peaks[service]["cgroup_lifetime_peak_bytes"] < peaks[service]["memory_limit_bytes"]
assert len({item[0]["id"] for item in uploads}) == 10
assert len({item[1] for item in uploads}) == 2
result["profiles"].append({
"name": name, "file_bytes": len(raw), "dimensions": dimensions, "successes": len(uploads),
"errors": 0, "seconds": round(time.monotonic() - started, 3),
"slowest_upload_seconds": round(max(item[2] for item in uploads), 3),
"replicas_used": 2, "originals_verified": 10, "memory": peaks,
})
for service in records:
data = json.loads(lab.run(args, ["docker", "inspect", "--format",
'{"oom":{{json .State.OOMKilled}},"restarts":{{.RestartCount}}}',
initial[service]["id"]], quiet=True))
assert not data["oom"] and data["restarts"] == 0
result["status"] = "PASS"
lab.evidence("memory.json", result)
print("PASS: 10 concurrent uploads in both profiles; originals intact, no OOM or restarts")
finally:
for service, process in samplers.items():
lab.compose(args, "exec", "-T", service, "python", "-c", "from pathlib import Path; Path(" + repr(flag) + ").touch()",
quiet=True)
process.wait(timeout=10)
for reader in readers:
reader.join(timeout=5)
lab.compose(args, "stop", "--timeout", "10", *RUNTIME)
if __name__ == "__main__":
main()