forked from mtusi2026/mtusi2026lab2
Bring the application, own tests, Dockerfile, toolchain, compose stack,
HAProxy, k6, RustFS and RabbitMQ configuration, migrations, hooks,
scenario scripts and the CI job from the lab 5 solution commit 64d5c67,
keep the lab 5 Makefile targets on top of the unchanged skeleton
targets, and append the lab 5 solution variables to the skeleton
.env.example. The package and the application follow the contract
version 1.6.0, and the image, the throwaway test containers and the
broker vhost description are tagged lab6 so they do not collide with a
lab 5 stand on the same host.
This is the starting point of the lab 6 cycle: lint, own tests and the
offline skeleton check pass before any lab 6 change.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
353 lines
12 KiB
Python
Executable File
353 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MAX_UPLOAD = 10 * 1024 * 1024
|
|
COLUMNS = (
|
|
"id",
|
|
"chat_id",
|
|
"owner_id",
|
|
"filename",
|
|
"content_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"width",
|
|
"height",
|
|
"object_key",
|
|
"status",
|
|
)
|
|
|
|
|
|
def chunk(tag, data):
|
|
return (
|
|
struct.pack(">I", len(data))
|
|
+ tag
|
|
+ data
|
|
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
)
|
|
|
|
|
|
def png(width, height):
|
|
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
|
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def sized_png(target):
|
|
height = max(1, target // (1024 * 3 + 1))
|
|
body = png(1024, height)
|
|
while len(body) > min(target, MAX_UPLOAD) and height > 1:
|
|
height -= max(1, (len(body) - target) // 3073 + 1)
|
|
body = png(1024, height)
|
|
return body
|
|
|
|
|
|
def run(cmd, stdin=None):
|
|
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
|
|
if out.returncode:
|
|
raise SystemExit(
|
|
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
|
|
)
|
|
return out.stdout.decode()
|
|
|
|
|
|
def psql(sql, stdin=None):
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
psql = [
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
]
|
|
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
|
|
|
|
|
|
def direct(args):
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
|
if args.chat_id and args.owner_id:
|
|
chat_id, owner_id = args.chat_id, args.owner_id
|
|
else:
|
|
found = psql(
|
|
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
|
|
).split()
|
|
if len(found) != 2:
|
|
raise SystemExit(
|
|
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
|
|
)
|
|
chat_id, owner_id = found
|
|
rows = []
|
|
started = time.monotonic()
|
|
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
|
|
Path(tmp).chmod(0o755)
|
|
for i in range(args.count):
|
|
aid = str(uuid.uuid4())
|
|
body = png(8, 8)
|
|
path = Path(tmp) / aid
|
|
path.write_bytes(body)
|
|
path.chmod(0o644)
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
rows.append(
|
|
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
|
|
)
|
|
generated = time.monotonic()
|
|
cli = [
|
|
"docker",
|
|
"compose",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
f"{tmp}:/seed:ro",
|
|
"s3-cli",
|
|
]
|
|
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
|
|
uploaded = time.monotonic()
|
|
columns = ", ".join(COLUMNS)
|
|
psql(
|
|
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
|
|
"".join(f"{r}\n" for r in rows).encode(),
|
|
)
|
|
seconds = {
|
|
"generate": round(generated - started, 3),
|
|
"s3_put": round(uploaded - generated, 3),
|
|
"pg_copy": round(time.monotonic() - uploaded, 3),
|
|
}
|
|
return {"mode": "direct", "count": args.count, "seconds": seconds}
|
|
|
|
|
|
class Http:
|
|
def __init__(self, base, timeout):
|
|
parts = urlsplit(base)
|
|
self.cls = (
|
|
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
)
|
|
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
|
|
|
|
def call(self, method, path, body=None, headers=None, token=None):
|
|
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
|
|
self.local.conn = conn
|
|
h = {"Accept": "application/json"}
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
if isinstance(body, dict):
|
|
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
|
|
h.update(headers or {})
|
|
try:
|
|
conn.request(method, path, body=body, headers=h)
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
except (OSError, http.client.HTTPException):
|
|
conn.close()
|
|
self.local.conn = None
|
|
return 0, {}, b""
|
|
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
|
|
|
|
|
|
def login(api, label):
|
|
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Seed-" + uuid.uuid4().hex
|
|
body = {"username": name, "display_name": label, "password": password}
|
|
status, _, data = api.call("POST", "/api/v1/users", body)
|
|
if status != 201:
|
|
raise SystemExit(f"user create failed: {status}")
|
|
user = json.loads(data)
|
|
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
|
|
status, _, data = api.call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
|
|
)
|
|
if status != 201:
|
|
raise SystemExit(f"login failed: {status}")
|
|
return user["id"], json.loads(data)["token"]
|
|
|
|
|
|
def mib(text):
|
|
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
|
|
if not m:
|
|
return 0.0
|
|
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
|
|
return float(m.group(1)) * scale.get(m.group(2), 1)
|
|
|
|
|
|
def sampler(ids, peaks, stop):
|
|
while not stop.is_set():
|
|
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
|
|
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
|
|
for line in out.splitlines():
|
|
cid, usage = line.split(" ", 1)
|
|
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
|
|
|
|
|
|
def cgroup(cid, name):
|
|
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
|
|
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
|
|
|
|
|
|
def percentile(values, q):
|
|
return (
|
|
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
|
|
if values
|
|
else None
|
|
)
|
|
|
|
|
|
def over_http(args):
|
|
api = Http(args.base_url, args.timeout)
|
|
alice, token = login(api, "alice")
|
|
bob, _ = login(api, "bob")
|
|
status, _, data = api.call(
|
|
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
|
|
)
|
|
if status != 201:
|
|
raise SystemExit(f"chat create failed: {status}")
|
|
chat = json.loads(data)["id"]
|
|
fixed = sized_png(args.size) if args.size else None
|
|
counter, lock, results = iter(range(args.count)), threading.Lock(), []
|
|
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
|
|
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
|
|
peaks, stop = {}, threading.Event()
|
|
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
|
|
if ids:
|
|
watcher.start()
|
|
|
|
def worker():
|
|
while True:
|
|
with lock:
|
|
i = next(counter, None)
|
|
if i is None:
|
|
return
|
|
body = fixed or png(8, 8)
|
|
began = time.monotonic()
|
|
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
|
|
status, headers, data = api.call(
|
|
"POST", path, body, {"Content-Type": "image/png"}, token
|
|
)
|
|
created = json.loads(data)["id"] if status == 201 else None
|
|
results.append(
|
|
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
|
|
)
|
|
|
|
started = time.monotonic()
|
|
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
elapsed = time.monotonic() - started
|
|
stop.set()
|
|
if ids:
|
|
watcher.join()
|
|
created = [r[3] for r in results if r[3]]
|
|
for start in range(0, min(args.link, len(created)), 10):
|
|
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
|
|
api.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat}/messages",
|
|
body,
|
|
{"Idempotency-Key": uuid.uuid4().hex},
|
|
token,
|
|
)
|
|
if args.ids_out:
|
|
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
|
|
codes, instances = {}, {}
|
|
for status, _, instance, _ in results:
|
|
codes[str(status)] = codes.get(str(status), 0) + 1
|
|
instances[instance] = instances.get(instance, 0) + 1
|
|
ok = [r[1] for r in results if r[0] == 201]
|
|
memory = {}
|
|
for cid in ids:
|
|
peak = cgroup(cid, "memory.peak")
|
|
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
|
|
memory[cid[:12]] = {
|
|
"idle_mib": idle[cid],
|
|
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
|
|
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
|
|
"oom_kill": int(events.get("oom_kill", 0)),
|
|
}
|
|
return {
|
|
"mode": "http",
|
|
"count": args.count,
|
|
"concurrency": args.concurrency,
|
|
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
|
|
"status_codes": codes,
|
|
"instances": len([i for i in instances if i]),
|
|
"linked_in_messages": min(args.link, len(created)),
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
|
|
"latency_ms": {
|
|
"p50": percentile(ok, 0.5),
|
|
"p95": percentile(ok, 0.95),
|
|
"p99": percentile(ok, 0.99),
|
|
"max": percentile(ok, 1.0),
|
|
},
|
|
"memory": memory,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
|
|
)
|
|
p.add_argument("--mode", choices=["http", "direct"], default="http")
|
|
p.add_argument("--count", type=int, default=1000)
|
|
p.add_argument("--concurrency", type=int, default=16)
|
|
p.add_argument(
|
|
"--size",
|
|
type=int,
|
|
default=0,
|
|
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
|
|
)
|
|
p.add_argument(
|
|
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
|
|
)
|
|
p.add_argument(
|
|
"--sample",
|
|
action="store_true",
|
|
help="http: sample docker stats and memory.peak of API containers",
|
|
)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
p.add_argument("--timeout", type=float, default=60)
|
|
p.add_argument("--ids-out", metavar="FILE")
|
|
p.add_argument("--chat-id")
|
|
p.add_argument("--owner-id")
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
report = direct(args) if args.mode == "direct" else over_http(args)
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|