forked from mtusi2026/mtusi2026lab4
382 lines
13 KiB
Python
382 lines
13 KiB
Python
import concurrent.futures
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.request
|
|
import zlib
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
from client import Client
|
|
|
|
LIMIT = 512 * 1024 * 1024
|
|
SIZE = 10485760
|
|
BASE_URL = "http://127.0.0.1:18080"
|
|
|
|
|
|
def command(args, **kwargs):
|
|
return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip()
|
|
|
|
|
|
def png(width, height):
|
|
def chunk(kind, data):
|
|
return (
|
|
struct.pack(">I", len(data))
|
|
+ kind
|
|
+ data
|
|
+ struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
|
)
|
|
|
|
compressor = zlib.compressobj(6)
|
|
row = b"\0" + bytes((20, 100, 50)) * width
|
|
compressed = (
|
|
b"".join(compressor.compress(row) for _ in range(height)) + compressor.flush()
|
|
)
|
|
data = (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
|
+ chunk(b"IDAT", compressed)
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
return data + b"\0" * (SIZE - len(data))
|
|
|
|
|
|
def inspect(ids):
|
|
data = json.loads(command(["docker", "inspect", *ids]))
|
|
return [
|
|
{
|
|
"id": item["Id"],
|
|
"started_at": item["State"]["StartedAt"],
|
|
"image": item["Image"],
|
|
"restarts": item["RestartCount"],
|
|
"oom_killed": item["State"]["OOMKilled"],
|
|
"memory_limit": item["HostConfig"]["Memory"],
|
|
"cpus": item["HostConfig"]["NanoCpus"],
|
|
}
|
|
for item in data
|
|
]
|
|
|
|
|
|
def cgroup(container):
|
|
text = command(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
container,
|
|
"sh",
|
|
"-c",
|
|
"cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory.events",
|
|
]
|
|
)
|
|
lines = text.splitlines()
|
|
return {
|
|
"current": int(lines[0]),
|
|
"peak": int(lines[1]),
|
|
"limit": int(lines[2]),
|
|
"events": {line.split()[0]: int(line.split()[1]) for line in lines[3:]},
|
|
}
|
|
|
|
|
|
class Sampler:
|
|
def __init__(self, ids, path):
|
|
self.file = path.open("w")
|
|
self.lock = threading.Lock()
|
|
self.processes = []
|
|
self.threads = []
|
|
self.counts = {cid: 0 for cid in ids}
|
|
self.errors = []
|
|
script = 'while true; do read current < /sys/fs/cgroup/memory.current; read peak < /sys/fs/cgroup/memory.peak; printf \'%s %s\\n\' "$current" "$peak"; sleep 0.1; done'
|
|
for cid in ids:
|
|
process = subprocess.Popen(
|
|
["docker", "exec", cid, "sh", "-c", script],
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
)
|
|
self.processes.append(process)
|
|
thread = threading.Thread(
|
|
target=self.read, args=(cid, process), daemon=True
|
|
)
|
|
thread.start()
|
|
self.threads.append(thread)
|
|
|
|
def read(self, cid, process):
|
|
try:
|
|
for line in process.stdout:
|
|
current, peak = map(int, line.split())
|
|
with self.lock:
|
|
self.counts[cid] += 1
|
|
self.file.write(
|
|
json.dumps(
|
|
{
|
|
"container": cid,
|
|
"monotonic_ns": time.monotonic_ns(),
|
|
"current": current,
|
|
"peak": peak,
|
|
}
|
|
)
|
|
+ "\n"
|
|
)
|
|
self.file.flush()
|
|
except Exception as error:
|
|
self.errors.append(str(error))
|
|
|
|
def close(self):
|
|
for process in self.processes:
|
|
process.terminate()
|
|
for process in self.processes:
|
|
try:
|
|
process.wait(timeout=5)
|
|
except subprocess.TimeoutExpired:
|
|
process.kill()
|
|
process.wait()
|
|
for thread in self.threads:
|
|
thread.join(timeout=5)
|
|
self.file.close()
|
|
|
|
|
|
def verify_content(actor, attachment, digest):
|
|
request = urllib.request.Request(
|
|
BASE_URL + "/api/v1/attachments/" + attachment["id"] + "/content",
|
|
headers={"Authorization": "Bearer " + actor["token"]},
|
|
)
|
|
checksum = hashlib.sha256()
|
|
size = 0
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
for chunk in iter(lambda: response.read(65536), b""):
|
|
checksum.update(chunk)
|
|
size += len(chunk)
|
|
if checksum.hexdigest() != digest or size != SIZE:
|
|
raise AssertionError("download differs from original")
|
|
|
|
|
|
def run_case(compose, profile, data, directory):
|
|
client = Client(BASE_URL, timeout=45)
|
|
alice, bob, _, chat = client.bootstrap()
|
|
path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=memory.png"
|
|
digest = hashlib.sha256(data).hexdigest()
|
|
for _ in range(2):
|
|
client.call(
|
|
"POST",
|
|
path,
|
|
201,
|
|
actor=alice,
|
|
raw=data,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
ids = command(compose + ["ps", "-q", "api"]).splitlines()
|
|
if len(ids) != 2:
|
|
raise AssertionError("expected two API containers")
|
|
before = inspect(ids)
|
|
baseline = {cid: cgroup(cid) for cid in ids}
|
|
(directory / "before.json").write_text(
|
|
json.dumps({"containers": before, "cgroups": baseline}, indent=2) + "\n"
|
|
)
|
|
sampler = Sampler(ids, directory / "samples.jsonl")
|
|
barrier = threading.Barrier(11)
|
|
results = []
|
|
|
|
def upload(index):
|
|
barrier.wait()
|
|
started = time.monotonic_ns()
|
|
try:
|
|
response = client.call(
|
|
"POST",
|
|
path,
|
|
201,
|
|
actor=alice,
|
|
raw=data,
|
|
headers={"Content-Type": "image/png"},
|
|
schema="Attachment",
|
|
)
|
|
return {
|
|
"client": index,
|
|
"started_ns": started,
|
|
"ended_ns": time.monotonic_ns(),
|
|
"ok": True,
|
|
"instance": response["headers"]["X-Instance-Id"],
|
|
"attachment": response["json"],
|
|
}
|
|
except Exception as error:
|
|
return {
|
|
"client": index,
|
|
"started_ns": started,
|
|
"ended_ns": time.monotonic_ns(),
|
|
"ok": False,
|
|
"error": str(error),
|
|
}
|
|
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool:
|
|
futures = [pool.submit(upload, index) for index in range(10)]
|
|
barrier.wait()
|
|
results = [future.result() for future in futures]
|
|
finally:
|
|
sampler.close()
|
|
# Save responses before probing containers, including when an OOM stopped one.
|
|
(directory / "requests.json").write_text(json.dumps(results, indent=2) + "\n")
|
|
after = inspect(ids)
|
|
(directory / "after.json").write_text(json.dumps(after, indent=2) + "\n")
|
|
final = {cid: cgroup(cid) for cid in ids}
|
|
for result in results:
|
|
if result["ok"]:
|
|
try:
|
|
verify_content(bob, result["attachment"], digest)
|
|
result["original_verified"] = True
|
|
except Exception as error:
|
|
result["original_verified"] = False
|
|
result["verification_error"] = str(error)
|
|
passed = (
|
|
len(results) == 10
|
|
and all(r["ok"] and r.get("original_verified") for r in results)
|
|
and len({r.get("instance") for r in results}) == 2
|
|
and before == after
|
|
and all(not r["oom_killed"] for r in after)
|
|
and all(
|
|
final[cid]["peak"] < LIMIT
|
|
and final[cid]["limit"] == LIMIT
|
|
and final[cid]["events"].get("oom_kill", 0)
|
|
== baseline[cid]["events"].get("oom_kill", 0)
|
|
and final[cid]["events"].get("oom", 0)
|
|
== baseline[cid]["events"].get("oom", 0)
|
|
for cid in ids
|
|
)
|
|
and all(count > 0 for count in sampler.counts.values())
|
|
and not sampler.errors
|
|
)
|
|
summary = {
|
|
"profile": profile,
|
|
"passed": passed,
|
|
"before": before,
|
|
"after": after,
|
|
"baseline": baseline,
|
|
"final": final,
|
|
"sample_counts": sampler.counts,
|
|
"sampler_errors": sampler.errors,
|
|
"requests": results,
|
|
}
|
|
(directory / "results.json").write_text(json.dumps(summary, indent=2) + "\n")
|
|
return {
|
|
"profile": profile,
|
|
"passed": passed,
|
|
"peaks_mib": [round(final[cid]["peak"] / 1048576, 2) for cid in ids],
|
|
}
|
|
|
|
|
|
def main():
|
|
import fcntl
|
|
|
|
directory = (
|
|
ROOT
|
|
/ ".memory/upload-memory"
|
|
/ datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|
|
)
|
|
directory.mkdir(parents=True)
|
|
lock = (ROOT / ".memory/upload-memory.lock").open("w")
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
fixtures = {"moderate": png(320, 200), "pixels40m": png(8000, 5000)}
|
|
files = (
|
|
[
|
|
ROOT / "compose.yml",
|
|
ROOT / "infra/compose.upload-memory.yml",
|
|
ROOT / "infra/minio.Dockerfile",
|
|
ROOT / "infra/haproxy.cfg",
|
|
ROOT / "src/Dockerfile",
|
|
ROOT / "src/go.mod",
|
|
ROOT / "src/go.sum",
|
|
Path(__file__),
|
|
]
|
|
+ sorted((ROOT / "src/backend").rglob("*.go"))
|
|
+ sorted((ROOT / "src/backend").rglob("*.sql"))
|
|
)
|
|
hashes = {
|
|
str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in files
|
|
}
|
|
protocol = {
|
|
"version": 1,
|
|
"hypothesis": "10 concurrent upload requests remain within 512 MiB per API with no OOM or restarts and all originals preserved",
|
|
"clients": 10,
|
|
"api_replicas": 2,
|
|
"cpu_per_api": 0.5,
|
|
"memory_limit_bytes": LIMIT,
|
|
"tmpfs_bytes": 16777216,
|
|
"active_uploads_per_replica": 1,
|
|
"sampling_interval_seconds": 0.1,
|
|
"repetitions": 3,
|
|
"warmup_uploads": 2,
|
|
"request_timeout_seconds": 45,
|
|
"model": "one simultaneous burst of 10 requests, no retries",
|
|
"order": [profile for _ in range(3) for profile in fixtures],
|
|
"fixtures": {
|
|
name: {"size": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
|
for name, data in fixtures.items()
|
|
},
|
|
"isolation": "fresh Compose project and volumes per run; same code/resources; no parallel fault tests",
|
|
"criteria": "all 60 measured uploads and downloads succeed; both replicas used; cgroup peak strictly below limit; no OOM/restart",
|
|
"insufficient_data": "sampler failure or missing samples; retain run as inconclusive, never discard errors/OOM",
|
|
"stop_rules": "complete all six runs even on failed request or OOM; no favorable early stop",
|
|
"limitations": "one host; concurrent HTTP requests with at most two decoders; tmpfs counted in cgroup; sampler overhead",
|
|
"source_sha256": hashes,
|
|
}
|
|
serialized = json.dumps(protocol, indent=2).encode() + b"\n"
|
|
(directory / "protocol.json").write_bytes(serialized)
|
|
(directory / "protocol.sha256").write_text(
|
|
hashlib.sha256(serialized).hexdigest() + "\n"
|
|
)
|
|
print("Protocol fixed:", directory, flush=True)
|
|
summaries = []
|
|
stamp = directory.name.lower().replace("t", "").replace("z", "")
|
|
for index, profile in enumerate(protocol["order"], 1):
|
|
run = directory / f"{index:02d}-{profile}"
|
|
run.mkdir()
|
|
project = f"highload-lab4-upload-memory-{stamp}-{index}"
|
|
compose = [
|
|
"docker",
|
|
"compose",
|
|
"--project-name",
|
|
project,
|
|
"--env-file",
|
|
".env",
|
|
"-f",
|
|
"compose.yml",
|
|
"-f",
|
|
"infra/compose.upload-memory.yml",
|
|
]
|
|
(run / "project.txt").write_text(project + "\n")
|
|
try:
|
|
with (run / "startup.log").open("w") as log:
|
|
subprocess.run(
|
|
compose + ["up", "-d", "--build", "--wait"],
|
|
cwd=ROOT,
|
|
stdout=log,
|
|
stderr=subprocess.STDOUT,
|
|
check=True,
|
|
)
|
|
summary = run_case(compose, profile, fixtures[profile], run)
|
|
except Exception as error:
|
|
summary = {"profile": profile, "passed": False, "error": str(error)}
|
|
(run / "failure.json").write_text(json.dumps(summary, indent=2) + "\n")
|
|
finally:
|
|
with (run / "shutdown.log").open("w") as log:
|
|
subprocess.run(
|
|
compose + ["down"], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT
|
|
)
|
|
summaries.append(summary)
|
|
print(json.dumps(summary), flush=True)
|
|
(directory / "summary.json").write_text(json.dumps(summaries, indent=2) + "\n")
|
|
if not all(case["passed"] for case in summaries):
|
|
raise SystemExit(1)
|
|
print("PASS: all six upload-memory runs", flush=True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|