Files
mtusi2026labs/scripts/lab4_seed.py
T
ArtyomandClaude Opus 5.5 a774a2fbbd Measure upload memory and bound the buffers along the chain
upload_memory.py recreates the replicas and the balancer for every run
of 10, 20 and 40 concurrent uploads of 1 and 10 MiB with 64 slots
(effectively no limit) and with 3, drives them through lab4_seed.py
from the prepared tooling, and keeps per-run JSON plus a summary with
the cgroup memory.peak of each replica and of the balancer and the
oom_kill counters. The seed script now also records memory.current
before the load and reads X-Instance-Id regardless of case.

The first run showed that slots alone only move a waiting body out of
the worker: with 3 slots and 40 uploads of 10 MiB the kernel grew the
socket buffers of HAProxy to about 110 MB and the 128 MiB balancer was
killed. HAProxy now caps its client and server socket buffers at 64
KiB and the API replicas cap tcp_rmem and tcp_wmem at 256 KiB, which
keeps the balancer at 30 MiB. glibc raised its mmap threshold after
the first large free and kept freed 10 MiB bodies in the heap, so the
API fixes MALLOC_MMAP_THRESHOLD_ at 128 KiB.

With these settings 40 concurrent uploads of 10 MiB peak at 323 MiB of
512 per replica and all 80 succeed, while without the slot limit the
same load reaches 512 MiB, a worker is killed and only 31 of 80
succeed. UPLOAD_CONCURRENCY stays at 3.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 00:14:09 +03:00

353 lines
12 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import base64
import hashlib
import http.client
import json
import os
import re
import struct
import subprocess
import tempfile
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
MAX_UPLOAD = 10 * 1024 * 1024
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
)
def chunk(tag, data):
return (
struct.pack(">I", len(data))
+ tag
+ data
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
)
def png(width, height):
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def sized_png(target):
height = max(1, target // (1024 * 3 + 1))
body = png(1024, height)
while len(body) > min(target, MAX_UPLOAD) and height > 1:
height -= max(1, (len(body) - target) // 3073 + 1)
body = png(1024, height)
return body
def run(cmd, stdin=None):
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
if out.returncode:
raise SystemExit(
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
)
return out.stdout.decode()
def psql(sql, stdin=None):
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-F",
" ",
]
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
def direct(args):
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
prefix = os.environ.get("S3_PREFIX", "attachments/")
if args.chat_id and args.owner_id:
chat_id, owner_id = args.chat_id, args.owner_id
else:
found = psql(
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
).split()
if len(found) != 2:
raise SystemExit(
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
)
chat_id, owner_id = found
rows = []
started = time.monotonic()
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
Path(tmp).chmod(0o755)
for i in range(args.count):
aid = str(uuid.uuid4())
body = png(8, 8)
path = Path(tmp) / aid
path.write_bytes(body)
path.chmod(0o644)
digest = hashlib.sha256(body).hexdigest()
rows.append(
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
)
generated = time.monotonic()
cli = [
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{tmp}:/seed:ro",
"s3-cli",
]
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
uploaded = time.monotonic()
columns = ", ".join(COLUMNS)
psql(
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
"".join(f"{r}\n" for r in rows).encode(),
)
seconds = {
"generate": round(generated - started, 3),
"s3_put": round(uploaded - generated, 3),
"pg_copy": round(time.monotonic() - uploaded, 3),
}
return {"mode": "direct", "count": args.count, "seconds": seconds}
class Http:
def __init__(self, base, timeout):
parts = urlsplit(base)
self.cls = (
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
)
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
def call(self, method, path, body=None, headers=None, token=None):
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
self.local.conn = conn
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if isinstance(body, dict):
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
h.update(headers or {})
try:
conn.request(method, path, body=body, headers=h)
response = conn.getresponse()
data = response.read()
except (OSError, http.client.HTTPException):
conn.close()
self.local.conn = None
return 0, {}, b""
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
def login(api, label):
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
password = "Seed-" + uuid.uuid4().hex
body = {"username": name, "display_name": label, "password": password}
status, _, data = api.call("POST", "/api/v1/users", body)
if status != 201:
raise SystemExit(f"user create failed: {status}")
user = json.loads(data)
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
status, _, data = api.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
)
if status != 201:
raise SystemExit(f"login failed: {status}")
return user["id"], json.loads(data)["token"]
def mib(text):
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
if not m:
return 0.0
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
return float(m.group(1)) * scale.get(m.group(2), 1)
def sampler(ids, peaks, stop):
while not stop.is_set():
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
for line in out.splitlines():
cid, usage = line.split(" ", 1)
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
def cgroup(cid, name):
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
def percentile(values, q):
return (
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
if values
else None
)
def over_http(args):
api = Http(args.base_url, args.timeout)
alice, token = login(api, "alice")
bob, _ = login(api, "bob")
status, _, data = api.call(
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
)
if status != 201:
raise SystemExit(f"chat create failed: {status}")
chat = json.loads(data)["id"]
fixed = sized_png(args.size) if args.size else None
counter, lock, results = iter(range(args.count)), threading.Lock(), []
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
peaks, stop = {}, threading.Event()
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
if ids:
watcher.start()
def worker():
while True:
with lock:
i = next(counter, None)
if i is None:
return
body = fixed or png(8, 8)
began = time.monotonic()
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
status, headers, data = api.call(
"POST", path, body, {"Content-Type": "image/png"}, token
)
created = json.loads(data)["id"] if status == 201 else None
results.append(
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
)
started = time.monotonic()
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
for t in threads:
t.start()
for t in threads:
t.join()
elapsed = time.monotonic() - started
stop.set()
if ids:
watcher.join()
created = [r[3] for r in results if r[3]]
for start in range(0, min(args.link, len(created)), 10):
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
api.call(
"POST",
f"/api/v1/chats/{chat}/messages",
body,
{"Idempotency-Key": uuid.uuid4().hex},
token,
)
if args.ids_out:
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
codes, instances = {}, {}
for status, _, instance, _ in results:
codes[str(status)] = codes.get(str(status), 0) + 1
instances[instance] = instances.get(instance, 0) + 1
ok = [r[1] for r in results if r[0] == 201]
memory = {}
for cid in ids:
peak = cgroup(cid, "memory.peak")
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
memory[cid[:12]] = {
"idle_mib": idle[cid],
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
"oom_kill": int(events.get("oom_kill", 0)),
}
return {
"mode": "http",
"count": args.count,
"concurrency": args.concurrency,
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
"status_codes": codes,
"instances": len([i for i in instances if i]),
"linked_in_messages": min(args.link, len(created)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
"latency_ms": {
"p50": percentile(ok, 0.5),
"p95": percentile(ok, 0.95),
"p99": percentile(ok, 0.99),
"max": percentile(ok, 1.0),
},
"memory": memory,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
)
p.add_argument("--mode", choices=["http", "direct"], default="http")
p.add_argument("--count", type=int, default=1000)
p.add_argument("--concurrency", type=int, default=16)
p.add_argument(
"--size",
type=int,
default=0,
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
)
p.add_argument(
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
)
p.add_argument(
"--sample",
action="store_true",
help="http: sample docker stats and memory.peak of API containers",
)
p.add_argument("--service", default="api")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--timeout", type=float, default=60)
p.add_argument("--ids-out", metavar="FILE")
p.add_argument("--chat-id")
p.add_argument("--owner-id")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
report = direct(args) if args.mode == "direct" else over_http(args)
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
if __name__ == "__main__":
main()