import concurrent.futures import datetime import hashlib import json import struct import subprocess import sys import threading import time import urllib.request import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "tests")) from client import Client LIMIT = 512 * 1024 * 1024 SIZE = 10485760 BASE_URL = "http://127.0.0.1:18080" def command(args, **kwargs): return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip() def png(width, height): def chunk(kind, data): return ( struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) ) compressor = zlib.compressobj(6) row = b"\0" + bytes((20, 100, 50)) * width compressed = ( b"".join(compressor.compress(row) for _ in range(height)) + compressor.flush() ) data = ( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + chunk(b"IDAT", compressed) + chunk(b"IEND", b"") ) return data + b"\0" * (SIZE - len(data)) def inspect(ids): data = json.loads(command(["docker", "inspect", *ids])) return [ { "id": item["Id"], "started_at": item["State"]["StartedAt"], "image": item["Image"], "restarts": item["RestartCount"], "oom_killed": item["State"]["OOMKilled"], "memory_limit": item["HostConfig"]["Memory"], "cpus": item["HostConfig"]["NanoCpus"], } for item in data ] def cgroup(container): text = command( [ "docker", "exec", container, "sh", "-c", "cat /sys/fs/cgroup/memory.current /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory.events", ] ) lines = text.splitlines() return { "current": int(lines[0]), "peak": int(lines[1]), "limit": int(lines[2]), "events": {line.split()[0]: int(line.split()[1]) for line in lines[3:]}, } class Sampler: def __init__(self, ids, path): self.file = path.open("w") self.lock = threading.Lock() self.processes = [] self.threads = [] self.counts = {cid: 0 for cid in ids} self.errors = [] script = 'while true; do read current < /sys/fs/cgroup/memory.current; read peak < /sys/fs/cgroup/memory.peak; printf \'%s %s\\n\' "$current" "$peak"; sleep 0.1; done' for cid in ids: process = subprocess.Popen( ["docker", "exec", cid, "sh", "-c", script], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) self.processes.append(process) thread = threading.Thread( target=self.read, args=(cid, process), daemon=True ) thread.start() self.threads.append(thread) def read(self, cid, process): try: for line in process.stdout: current, peak = map(int, line.split()) with self.lock: self.counts[cid] += 1 self.file.write( json.dumps( { "container": cid, "monotonic_ns": time.monotonic_ns(), "current": current, "peak": peak, } ) + "\n" ) self.file.flush() except Exception as error: self.errors.append(str(error)) def close(self): for process in self.processes: process.terminate() for process in self.processes: try: process.wait(timeout=5) except subprocess.TimeoutExpired: process.kill() process.wait() for thread in self.threads: thread.join(timeout=5) self.file.close() def verify_content(actor, attachment, digest): request = urllib.request.Request( BASE_URL + "/api/v1/attachments/" + attachment["id"] + "/content", headers={"Authorization": "Bearer " + actor["token"]}, ) checksum = hashlib.sha256() size = 0 with urllib.request.urlopen(request, timeout=30) as response: for chunk in iter(lambda: response.read(65536), b""): checksum.update(chunk) size += len(chunk) if checksum.hexdigest() != digest or size != SIZE: raise AssertionError("download differs from original") def run_case(compose, profile, data, directory): client = Client(BASE_URL, timeout=45) alice, bob, _, chat = client.bootstrap() path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=memory.png" digest = hashlib.sha256(data).hexdigest() for _ in range(2): client.call( "POST", path, 201, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", ) ids = command(compose + ["ps", "-q", "api"]).splitlines() if len(ids) != 2: raise AssertionError("expected two API containers") before = inspect(ids) baseline = {cid: cgroup(cid) for cid in ids} (directory / "before.json").write_text( json.dumps({"containers": before, "cgroups": baseline}, indent=2) + "\n" ) sampler = Sampler(ids, directory / "samples.jsonl") barrier = threading.Barrier(11) results = [] def upload(index): barrier.wait() started = time.monotonic_ns() try: response = client.call( "POST", path, 201, actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment", ) return { "client": index, "started_ns": started, "ended_ns": time.monotonic_ns(), "ok": True, "instance": response["headers"]["X-Instance-Id"], "attachment": response["json"], } except Exception as error: return { "client": index, "started_ns": started, "ended_ns": time.monotonic_ns(), "ok": False, "error": str(error), } try: with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: futures = [pool.submit(upload, index) for index in range(10)] barrier.wait() results = [future.result() for future in futures] finally: sampler.close() # Save responses before probing containers, including when an OOM stopped one. (directory / "requests.json").write_text(json.dumps(results, indent=2) + "\n") after = inspect(ids) (directory / "after.json").write_text(json.dumps(after, indent=2) + "\n") final = {cid: cgroup(cid) for cid in ids} for result in results: if result["ok"]: try: verify_content(bob, result["attachment"], digest) result["original_verified"] = True except Exception as error: result["original_verified"] = False result["verification_error"] = str(error) passed = ( len(results) == 10 and all(r["ok"] and r.get("original_verified") for r in results) and len({r.get("instance") for r in results}) == 2 and before == after and all(not r["oom_killed"] for r in after) and all( final[cid]["peak"] < LIMIT and final[cid]["limit"] == LIMIT and final[cid]["events"].get("oom_kill", 0) == baseline[cid]["events"].get("oom_kill", 0) and final[cid]["events"].get("oom", 0) == baseline[cid]["events"].get("oom", 0) for cid in ids ) and all(count > 0 for count in sampler.counts.values()) and not sampler.errors ) summary = { "profile": profile, "passed": passed, "before": before, "after": after, "baseline": baseline, "final": final, "sample_counts": sampler.counts, "sampler_errors": sampler.errors, "requests": results, } (directory / "results.json").write_text(json.dumps(summary, indent=2) + "\n") return { "profile": profile, "passed": passed, "peaks_mib": [round(final[cid]["peak"] / 1048576, 2) for cid in ids], } def main(): import fcntl directory = ( ROOT / ".memory/upload-memory" / datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") ) directory.mkdir(parents=True) lock = (ROOT / ".memory/upload-memory.lock").open("w") fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) fixtures = {"moderate": png(320, 200), "pixels40m": png(8000, 5000)} files = ( [ ROOT / "compose.yml", ROOT / "infra/compose.upload-memory.yml", ROOT / "infra/minio.Dockerfile", ROOT / "infra/haproxy.cfg", ROOT / "src/Dockerfile", ROOT / "src/go.mod", ROOT / "src/go.sum", Path(__file__), ] + sorted((ROOT / "src/backend").rglob("*.go")) + sorted((ROOT / "src/backend").rglob("*.sql")) ) hashes = { str(path.relative_to(ROOT)): hashlib.sha256(path.read_bytes()).hexdigest() for path in files } protocol = { "version": 1, "hypothesis": "10 concurrent upload requests remain within 512 MiB per API with no OOM or restarts and all originals preserved", "clients": 10, "api_replicas": 2, "cpu_per_api": 0.5, "memory_limit_bytes": LIMIT, "tmpfs_bytes": 16777216, "active_uploads_per_replica": 1, "sampling_interval_seconds": 0.1, "repetitions": 3, "warmup_uploads": 2, "request_timeout_seconds": 45, "model": "one simultaneous burst of 10 requests, no retries", "order": [profile for _ in range(3) for profile in fixtures], "fixtures": { name: {"size": len(data), "sha256": hashlib.sha256(data).hexdigest()} for name, data in fixtures.items() }, "isolation": "fresh Compose project and volumes per run; same code/resources; no parallel fault tests", "criteria": "all 60 measured uploads and downloads succeed; both replicas used; cgroup peak strictly below limit; no OOM/restart", "insufficient_data": "sampler failure or missing samples; retain run as inconclusive, never discard errors/OOM", "stop_rules": "complete all six runs even on failed request or OOM; no favorable early stop", "limitations": "one host; concurrent HTTP requests with at most two decoders; tmpfs counted in cgroup; sampler overhead", "source_sha256": hashes, } serialized = json.dumps(protocol, indent=2).encode() + b"\n" (directory / "protocol.json").write_bytes(serialized) (directory / "protocol.sha256").write_text( hashlib.sha256(serialized).hexdigest() + "\n" ) print("Protocol fixed:", directory, flush=True) summaries = [] stamp = directory.name.lower().replace("t", "").replace("z", "") for index, profile in enumerate(protocol["order"], 1): run = directory / f"{index:02d}-{profile}" run.mkdir() project = f"highload-lab4-upload-memory-{stamp}-{index}" compose = [ "docker", "compose", "--project-name", project, "--env-file", ".env", "-f", "compose.yml", "-f", "infra/compose.upload-memory.yml", ] (run / "project.txt").write_text(project + "\n") try: with (run / "startup.log").open("w") as log: subprocess.run( compose + ["up", "-d", "--build", "--wait"], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT, check=True, ) summary = run_case(compose, profile, fixtures[profile], run) except Exception as error: summary = {"profile": profile, "passed": False, "error": str(error)} (run / "failure.json").write_text(json.dumps(summary, indent=2) + "\n") finally: with (run / "shutdown.log").open("w") as log: subprocess.run( compose + ["down"], cwd=ROOT, stdout=log, stderr=subprocess.STDOUT ) summaries.append(summary) print(json.dumps(summary), flush=True) (directory / "summary.json").write_text(json.dumps(summaries, indent=2) + "\n") if not all(case["passed"] for case in summaries): raise SystemExit(1) print("PASS: all six upload-memory runs", flush=True) if __name__ == "__main__": main()