Files
mtusi2026lab4-fork/scripts/check_upload_memory.py

382 lines
13 KiB
Python

import concurrent.futures
import datetime
import hashlib
import json
import struct
import subprocess
import sys
import threading
import time
import urllib.request
import zlib
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
LIMIT = 512 * 1024 * 1024
SIZE = 10485760
BASE_URL = "http://127.0.0.1:18080"
def command(args, **kwargs):
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
def png(width, height):
def chunk(kind, data):
return (
struct.pack(">I", len(data))
+ kind
+ data
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
)
compressor = zlib.compressobj(6)
row = b"\0" + bytes((20, 100, 50)) * width
compressed = (
b"".join(compressor.compress(row) for _ in range(height)) + compressor.flush()
)
data = (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", compressed)
+ chunk(b"IEND", b"")
)
return data + b"\0" * (SIZE - len(data))
def inspect(ids):
data = json.loads(command(["docker", "inspect", *ids]))
return [
{
"id": item["Id"],
"started_at": item["State"]["StartedAt"],
"image": item["Image"],
"restarts": item["RestartCount"],
"oom_killed": item["State"]["OOMKilled"],
"memory_limit": item["HostConfig"]["Memory"],
"cpus": item["HostConfig"]["NanoCpus"],
}
for item in data
]
def cgroup(container):
text = command(
[
"docker",
"exec",
container,
"sh",
"-c",
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory.events",
]
)
lines = text.splitlines()
return {
"current": int(lines[0]),
"peak": int(lines[1]),
"limit": int(lines[2]),
"events": {line.split()[0]: int(line.split()[1]) for line in lines[3:]},
}
class Sampler:
def __init__(self, ids, path):
self.file = path.open("w")
self.lock = threading.Lock()
self.processes = []
self.threads = []
self.counts = {cid: 0 for cid in ids}
self.errors = []
script = 'while true; do read current < /sys/fs/cgroup/memory.current; read peak < /sys/fs/cgroup/memory.peak; printf \'%s %s\\n\' "$current" "$peak"; sleep 0.1; done'
for cid in ids:
process = subprocess.Popen(
["docker", "exec", cid, "sh", "-c", script],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
)
self.processes.append(process)
thread = threading.Thread(
target=self.read, args=(cid, process), daemon=True
)
thread.start()
self.threads.append(thread)
def read(self, cid, process):
try:
for line in process.stdout:
current, peak = map(int, line.split())
with self.lock:
self.counts[cid] += 1
self.file.write(
json.dumps(
{
"container": cid,
"monotonic_ns": time.monotonic_ns(),
"current": current,
"peak": peak,
}
)
+ "\n"
)
self.file.flush()
except Exception as error:
self.errors.append(str(error))
def close(self):
for process in self.processes:
process.terminate()
for process in self.processes:
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
for thread in self.threads:
thread.join(timeout=5)
self.file.close()
def verify_content(actor, attachment, digest):
request = urllib.request.Request(
BASE_URL + "/api/v1/attachments/" + attachment["id"] + "/content",
headers={"Authorization": "Bearer " + actor["token"]},
)
checksum = hashlib.sha256()
size = 0
with urllib.request.urlopen(request, timeout=30) as response:
for chunk in iter(lambda: response.read(65536), b""):
checksum.update(chunk)
size += len(chunk)
if checksum.hexdigest() != digest or size != SIZE:
raise AssertionError("download differs from original")
def run_case(compose, profile, data, directory):
client = Client(BASE_URL, timeout=45)
alice, bob, _, chat = client.bootstrap()
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=memory.png"
digest = hashlib.sha256(data).hexdigest()
for _ in range(2):
client.call(
"POST",
path,
201,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
ids = command(compose + ["ps", "-q", "api"]).splitlines()
if len(ids) != 2:
raise AssertionError("expected two API containers")
before = inspect(ids)
baseline = {cid: cgroup(cid) for cid in ids}
(directory / "before.json").write_text(
json.dumps({"containers": before, "cgroups": baseline}, indent=2) + "\n"
)
sampler = Sampler(ids, directory / "samples.jsonl")
barrier = threading.Barrier(11)
results = []
def upload(index):
barrier.wait()
started = time.monotonic_ns()
try:
response = client.call(
"POST",
path,
201,
actor=alice,
raw=data,
headers={"Content-Type": "image/png"},
schema="Attachment",
)
return {
"client": index,
"started_ns": started,
"ended_ns": time.monotonic_ns(),
"ok": True,
"instance": response["headers"]["X-Instance-Id"],
"attachment": response["json"],
}
except Exception as error:
return {
"client": index,
"started_ns": started,
"ended_ns": time.monotonic_ns(),
"ok": False,
"error": str(error),
}
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
futures = [pool.submit(upload, index) for index in range(10)]
barrier.wait()
results = [future.result() for future in futures]
finally:
sampler.close()
# Save responses before probing containers, including when an OOM stopped one.
(directory / "requests.json").write_text(json.dumps(results, indent=2) + "\n")
after = inspect(ids)
(directory / "after.json").write_text(json.dumps(after, indent=2) + "\n")
final = {cid: cgroup(cid) for cid in ids}
for result in results:
if result["ok"]:
try:
verify_content(bob, result["attachment"], digest)
result["original_verified"] = True
except Exception as error:
result["original_verified"] = False
result["verification_error"] = str(error)
passed = (
len(results) == 10
and all(r["ok"] and r.get("original_verified") for r in results)
and len({r.get("instance") for r in results}) == 2
and before == after
and all(not r["oom_killed"] for r in after)
and all(
final[cid]["peak"] < LIMIT
and final[cid]["limit"] == LIMIT
and final[cid]["events"].get("oom_kill", 0)
== baseline[cid]["events"].get("oom_kill", 0)
and final[cid]["events"].get("oom", 0)
== baseline[cid]["events"].get("oom", 0)
for cid in ids
)
and all(count > 0 for count in sampler.counts.values())
and not sampler.errors
)
summary = {
"profile": profile,
"passed": passed,
"before": before,
"after": after,
"baseline": baseline,
"final": final,
"sample_counts": sampler.counts,
"sampler_errors": sampler.errors,
"requests": results,
}
(directory / "results.json").write_text(json.dumps(summary, indent=2) + "\n")
return {
"profile": profile,
"passed": passed,
"peaks_mib": [round(final[cid]["peak"] / 1048576, 2) for cid in ids],
}
def main():
import fcntl
directory = (
ROOT
/ ".memory/upload-memory"
/ datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
)
directory.mkdir(parents=True)
lock = (ROOT / ".memory/upload-memory.lock").open("w")
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
fixtures = {"moderate": png(320, 200), "pixels40m": png(8000, 5000)}
files = (
[
ROOT / "compose.yml",
ROOT / "infra/compose.upload-memory.yml",
ROOT / "infra/minio.Dockerfile",
ROOT / "infra/haproxy.cfg",
ROOT / "src/Dockerfile",
ROOT / "src/go.mod",
ROOT / "src/go.sum",
Path(__file__),
]
+ sorted((ROOT / "src/backend").rglob("*.go"))
+ sorted((ROOT / "src/backend").rglob("*.sql"))
)
hashes = {
str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
for path in files
}
protocol = {
"version": 1,
"hypothesis": "10 concurrent upload requests remain within 512 MiB per API with no OOM or restarts and all originals preserved",
"clients": 10,
"api_replicas": 2,
"cpu_per_api": 0.5,
"memory_limit_bytes": LIMIT,
"tmpfs_bytes": 16777216,
"active_uploads_per_replica": 1,
"sampling_interval_seconds": 0.1,
"repetitions": 3,
"warmup_uploads": 2,
"request_timeout_seconds": 45,
"model": "one simultaneous burst of 10 requests, no retries",
"order": [profile for _ in range(3) for profile in fixtures],
"fixtures": {
name: {"size": len(data), "sha256": hashlib.sha256(data).hexdigest()}
for name, data in fixtures.items()
},
"isolation": "fresh Compose project and volumes per run; same code/resources; no parallel fault tests",
"criteria": "all 60 measured uploads and downloads succeed; both replicas used; cgroup peak strictly below limit; no OOM/restart",
"insufficient_data": "sampler failure or missing samples; retain run as inconclusive, never discard errors/OOM",
"stop_rules": "complete all six runs even on failed request or OOM; no favorable early stop",
"limitations": "one host; concurrent HTTP requests with at most two decoders; tmpfs counted in cgroup; sampler overhead",
"source_sha256": hashes,
}
serialized = json.dumps(protocol, indent=2).encode() + b"\n"
(directory / "protocol.json").write_bytes(serialized)
(directory / "protocol.sha256").write_text(
hashlib.sha256(serialized).hexdigest() + "\n"
)
print("Protocol fixed:", directory, flush=True)
summaries = []
stamp = directory.name.lower().replace("t", "").replace("z", "")
for index, profile in enumerate(protocol["order"], 1):
run = directory / f"{index:02d}-{profile}"
run.mkdir()
project = f"highload-lab4-upload-memory-{stamp}-{index}"
compose = [
"docker",
"compose",
"--project-name",
project,
"--env-file",
".env",
"-f",
"compose.yml",
"-f",
"infra/compose.upload-memory.yml",
]
(run / "project.txt").write_text(project + "\n")
try:
with (run / "startup.log").open("w") as log:
subprocess.run(
compose + ["up", "-d", "--build", "--wait"],
cwd=ROOT,
stdout=log,
stderr=subprocess.STDOUT,
check=True,
)
summary = run_case(compose, profile, fixtures[profile], run)
except Exception as error:
summary = {"profile": profile, "passed": False, "error": str(error)}
(run / "failure.json").write_text(json.dumps(summary, indent=2) + "\n")
finally:
with (run / "shutdown.log").open("w") as log:
subprocess.run(
compose + ["down"], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT
)
summaries.append(summary)
print(json.dumps(summary), flush=True)
(directory / "summary.json").write_text(json.dumps(summaries, indent=2) + "\n")
if not all(case["passed"] for case in summaries):
raise SystemExit(1)
print("PASS: all six upload-memory runs", flush=True)
if __name__ == "__main__":
main()