Files
mtusi2026lab6/scripts/lab4_seed.py
T
ArtyomandClaude Opus 5.5 5ae3d77152 Carry over the lab 5 solution
Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks,
scenario scripts and the CI job from the lab 5 solution commit 64d5c67,
keep the lab 5 Makefile targets on top of the unchanged skeleton
targets, and append the lab 5 solution variables to the skeleton
.env.example. The package and the application follow the contract
version 1.6.0, and the image, the throwaway test containers and the
broker vhost description are tagged lab6 so they do not collide with a
lab 5 stand on the same host.

This is the starting point of the lab 6 cycle: lint, own tests and the
offline skeleton check pass before any lab 6 change.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-29 03:51:58 +03:00

353 lines
12 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import base64
import hashlib
import http.client
import json
import os
import re
import struct
import subprocess
import tempfile
import threading
import time
import uuid
import zlib
from pathlib import Path
from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
MAX_UPLOAD = 10 * 1024 * 1024
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
)
def chunk(tag, data):
return (
struct.pack(">I", len(data))
+ tag
+ data
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
)
def png(width, height):
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return (
b"\x89PNG\r\n\x1a\n"
+ chunk(b"IHDR", header)
+ chunk(b"IDAT", zlib.compress(raw, 1))
+ chunk(b"IEND", b"")
)
def sized_png(target):
height = max(1, target // (1024 * 3 + 1))
body = png(1024, height)
while len(body) > min(target, MAX_UPLOAD) and height > 1:
height -= max(1, (len(body) - target) // 3073 + 1)
body = png(1024, height)
return body
def run(cmd, stdin=None):
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
if out.returncode:
raise SystemExit(
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
)
return out.stdout.decode()
def psql(sql, stdin=None):
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
psql = [
"docker",
"compose",
"exec",
"-T",
"postgres",
"psql",
"-v",
"ON_ERROR_STOP=1",
"-qAt",
"-F",
" ",
]
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
def direct(args):
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
prefix = os.environ.get("S3_PREFIX", "attachments/")
if args.chat_id and args.owner_id:
chat_id, owner_id = args.chat_id, args.owner_id
else:
found = psql(
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
).split()
if len(found) != 2:
raise SystemExit(
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
)
chat_id, owner_id = found
rows = []
started = time.monotonic()
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
Path(tmp).chmod(0o755)
for i in range(args.count):
aid = str(uuid.uuid4())
body = png(8, 8)
path = Path(tmp) / aid
path.write_bytes(body)
path.chmod(0o644)
digest = hashlib.sha256(body).hexdigest()
rows.append(
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
)
generated = time.monotonic()
cli = [
"docker",
"compose",
"run",
"--rm",
"--no-deps",
"-T",
"-v",
f"{tmp}:/seed:ro",
"s3-cli",
]
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
uploaded = time.monotonic()
columns = ", ".join(COLUMNS)
psql(
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
"".join(f"{r}\n" for r in rows).encode(),
)
seconds = {
"generate": round(generated - started, 3),
"s3_put": round(uploaded - generated, 3),
"pg_copy": round(time.monotonic() - uploaded, 3),
}
return {"mode": "direct", "count": args.count, "seconds": seconds}
class Http:
def __init__(self, base, timeout):
parts = urlsplit(base)
self.cls = (
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
)
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
def call(self, method, path, body=None, headers=None, token=None):
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
self.local.conn = conn
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if isinstance(body, dict):
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
h.update(headers or {})
try:
conn.request(method, path, body=body, headers=h)
response = conn.getresponse()
data = response.read()
except (OSError, http.client.HTTPException):
conn.close()
self.local.conn = None
return 0, {}, b""
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
def login(api, label):
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
password = "Seed-" + uuid.uuid4().hex
body = {"username": name, "display_name": label, "password": password}
status, _, data = api.call("POST", "/api/v1/users", body)
if status != 201:
raise SystemExit(f"user create failed: {status}")
user = json.loads(data)
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
status, _, data = api.call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
)
if status != 201:
raise SystemExit(f"login failed: {status}")
return user["id"], json.loads(data)["token"]
def mib(text):
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
if not m:
return 0.0
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
return float(m.group(1)) * scale.get(m.group(2), 1)
def sampler(ids, peaks, stop):
while not stop.is_set():
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
for line in out.splitlines():
cid, usage = line.split(" ", 1)
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
def cgroup(cid, name):
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
def percentile(values, q):
return (
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
if values
else None
)
def over_http(args):
api = Http(args.base_url, args.timeout)
alice, token = login(api, "alice")
bob, _ = login(api, "bob")
status, _, data = api.call(
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
)
if status != 201:
raise SystemExit(f"chat create failed: {status}")
chat = json.loads(data)["id"]
fixed = sized_png(args.size) if args.size else None
counter, lock, results = iter(range(args.count)), threading.Lock(), []
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
peaks, stop = {}, threading.Event()
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
if ids:
watcher.start()
def worker():
while True:
with lock:
i = next(counter, None)
if i is None:
return
body = fixed or png(8, 8)
began = time.monotonic()
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
status, headers, data = api.call(
"POST", path, body, {"Content-Type": "image/png"}, token
)
created = json.loads(data)["id"] if status == 201 else None
results.append(
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
)
started = time.monotonic()
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
for t in threads:
t.start()
for t in threads:
t.join()
elapsed = time.monotonic() - started
stop.set()
if ids:
watcher.join()
created = [r[3] for r in results if r[3]]
for start in range(0, min(args.link, len(created)), 10):
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
api.call(
"POST",
f"/api/v1/chats/{chat}/messages",
body,
{"Idempotency-Key": uuid.uuid4().hex},
token,
)
if args.ids_out:
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
codes, instances = {}, {}
for status, _, instance, _ in results:
codes[str(status)] = codes.get(str(status), 0) + 1
instances[instance] = instances.get(instance, 0) + 1
ok = [r[1] for r in results if r[0] == 201]
memory = {}
for cid in ids:
peak = cgroup(cid, "memory.peak")
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
memory[cid[:12]] = {
"idle_mib": idle[cid],
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
"oom_kill": int(events.get("oom_kill", 0)),
}
return {
"mode": "http",
"count": args.count,
"concurrency": args.concurrency,
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
"status_codes": codes,
"instances": len([i for i in instances if i]),
"linked_in_messages": min(args.link, len(created)),
"elapsed_seconds": round(elapsed, 3),
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
"latency_ms": {
"p50": percentile(ok, 0.5),
"p95": percentile(ok, 0.95),
"p99": percentile(ok, 0.99),
"max": percentile(ok, 1.0),
},
"memory": memory,
}
def main():
p = argparse.ArgumentParser(
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
)
p.add_argument("--mode", choices=["http", "direct"], default="http")
p.add_argument("--count", type=int, default=1000)
p.add_argument("--concurrency", type=int, default=16)
p.add_argument(
"--size",
type=int,
default=0,
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
)
p.add_argument(
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
)
p.add_argument(
"--sample",
action="store_true",
help="http: sample docker stats and memory.peak of API containers",
)
p.add_argument("--service", default="api")
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
p.add_argument("--timeout", type=float, default=60)
p.add_argument("--ids-out", metavar="FILE")
p.add_argument("--chat-id")
p.add_argument("--owner-id")
p.add_argument("--output", metavar="FILE")
args = p.parse_args()
report = direct(args) if args.mode == "direct" else over_http(args)
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
Path(args.output).write_text(text + "\n")
print(text)
if __name__ == "__main__":
main()