#!/usr/bin/env python3 import argparse import base64 import hashlib import http.client import json import os import re import struct import subprocess import tempfile import threading import time import uuid import zlib from pathlib import Path from urllib.parse import urlsplit ROOT = Path(__file__).resolve().parents[1] MAX_UPLOAD = 10 * 1024 * 1024 COLUMNS = ( "id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "width", "height", "object_key", "status", ) def chunk(tag, data): return ( struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) ) def png(width, height): raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) return ( b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"") ) def sized_png(target): height = max(1, target // (1024 * 3 + 1)) body = png(1024, height) while len(body) > min(target, MAX_UPLOAD) and height > 1: height -= max(1, (len(body) - target) // 3073 + 1) body = png(1024, height) return body def run(cmd, stdin=None): out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False) if out.returncode: raise SystemExit( f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}" ) return out.stdout.decode() def psql(sql, stdin=None): user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger") psql = [ "docker", "compose", "exec", "-T", "postgres", "psql", "-v", "ON_ERROR_STOP=1", "-qAt", "-F", " ", ] return run([*psql, "-U", user, "-d", db, "-c", sql], stdin) def direct(args): bucket = os.environ.get("S3_BUCKET", "messenger-attachments") prefix = os.environ.get("S3_PREFIX", "attachments/") if args.chat_id and args.owner_id: chat_id, owner_id = args.chat_id, args.owner_id else: found = psql( "SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1" ).split() if len(found) != 2: raise SystemExit( "no attachment rows yet: upload one via API or pass --chat-id and --owner-id" ) chat_id, owner_id = found rows = [] started = time.monotonic() with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp: Path(tmp).chmod(0o755) for i in range(args.count): aid = str(uuid.uuid4()) body = png(8, 8) path = Path(tmp) / aid path.write_bytes(body) path.chmod(0o644) digest = hashlib.sha256(body).hexdigest() rows.append( f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready" ) generated = time.monotonic() cli = [ "docker", "compose", "run", "--rm", "--no-deps", "-T", "-v", f"{tmp}:/seed:ro", "s3-cli", ] run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"]) uploaded = time.monotonic() columns = ", ".join(COLUMNS) psql( f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)", "".join(f"{r}\n" for r in rows).encode(), ) seconds = { "generate": round(generated - started, 3), "s3_put": round(uploaded - generated, 3), "pg_copy": round(time.monotonic() - uploaded, 3), } return {"mode": "direct", "count": args.count, "seconds": seconds} class Http: def __init__(self, base, timeout): parts = urlsplit(base) self.cls = ( http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection ) self.host, self.timeout, self.local = parts.netloc, timeout, threading.local() def call(self, method, path, body=None, headers=None, token=None): conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout) self.local.conn = conn h = {"Accept": "application/json"} if token: h["Authorization"] = "Bearer " + token if isinstance(body, dict): body, h["Content-Type"] = json.dumps(body).encode(), "application/json" h.update(headers or {}) try: conn.request(method, path, body=body, headers=h) response = conn.getresponse() data = response.read() except (OSError, http.client.HTTPException): conn.close() self.local.conn = None return 0, {}, b"" return response.status, {k.lower(): v for k, v in response.getheaders()}, data def login(api, label): name = f"seed_{label}_{uuid.uuid4().hex[:12]}" password = "Seed-" + uuid.uuid4().hex body = {"username": name, "display_name": label, "password": password} status, _, data = api.call("POST", "/api/v1/users", body) if status != 201: raise SystemExit(f"user create failed: {status}") user = json.loads(data) basic = base64.b64encode(f"{name}:{password}".encode()).decode() status, _, data = api.call( "POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"} ) if status != 201: raise SystemExit(f"login failed: {status}") return user["id"], json.loads(data)["token"] def mib(text): m = re.match(r"([\d.]+)\s*([KMG]i?B)", text) if not m: return 0.0 scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024} return float(m.group(1)) * scale.get(m.group(2), 1) def sampler(ids, peaks, stop): while not stop.is_set(): cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids] out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout for line in out.splitlines(): cid, usage = line.split(" ", 1) peaks[cid] = max(peaks.get(cid, 0.0), mib(usage)) def cgroup(cid, name): cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"] return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip() def percentile(values, q): return ( round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1) if values else None ) def over_http(args): api = Http(args.base_url, args.timeout) alice, token = login(api, "alice") bob, _ = login(api, "bob") status, _, data = api.call( "POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token ) if status != 201: raise SystemExit(f"chat create failed: {status}") chat = json.loads(data)["id"] fixed = sized_png(args.size) if args.size else None counter, lock, results = iter(range(args.count)), threading.Lock(), [] ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else [] idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids} peaks, stop = {}, threading.Event() watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True) if ids: watcher.start() def worker(): while True: with lock: i = next(counter, None) if i is None: return body = fixed or png(8, 8) began = time.monotonic() path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png" status, headers, data = api.call( "POST", path, body, {"Content-Type": "image/png"}, token ) created = json.loads(data)["id"] if status == 201 else None results.append( (status, time.monotonic() - began, headers.get("x-instance-id", ""), created) ) started = time.monotonic() threads = [threading.Thread(target=worker) for _ in range(args.concurrency)] for t in threads: t.start() for t in threads: t.join() elapsed = time.monotonic() - started stop.set() if ids: watcher.join() created = [r[3] for r in results if r[3]] for start in range(0, min(args.link, len(created)), 10): body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]} api.call( "POST", f"/api/v1/chats/{chat}/messages", body, {"Idempotency-Key": uuid.uuid4().hex}, token, ) if args.ids_out: Path(args.ids_out).write_text("".join(f"{i}\n" for i in created)) codes, instances = {}, {} for status, _, instance, _ in results: codes[str(status)] = codes.get(str(status), 0) + 1 instances[instance] = instances.get(instance, 0) + 1 ok = [r[1] for r in results if r[0] == 201] memory = {} for cid in ids: peak = cgroup(cid, "memory.peak") events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines()) memory[cid[:12]] = { "idle_mib": idle[cid], "docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1), "cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None, "oom_kill": int(events.get("oom_kill", 0)), } return { "mode": "http", "count": args.count, "concurrency": args.concurrency, "body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)", "status_codes": codes, "instances": len([i for i in instances if i]), "linked_in_messages": min(args.link, len(created)), "elapsed_seconds": round(elapsed, 3), "successful_rps": round(len(ok) / elapsed, 2) if elapsed else None, "latency_ms": { "p50": percentile(ok, 0.5), "p95": percentile(ok, 0.95), "p99": percentile(ok, 0.99), "max": percentile(ok, 1.0), }, "memory": memory, } def main(): p = argparse.ArgumentParser( description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding" ) p.add_argument("--mode", choices=["http", "direct"], default="http") p.add_argument("--count", type=int, default=1000) p.add_argument("--concurrency", type=int, default=16) p.add_argument( "--size", type=int, default=0, help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs", ) p.add_argument( "--link", type=int, default=0, help="http: reference the first N attachments from messages" ) p.add_argument( "--sample", action="store_true", help="http: sample docker stats and memory.peak of API containers", ) p.add_argument("--service", default="api") p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) p.add_argument("--timeout", type=float, default=60) p.add_argument("--ids-out", metavar="FILE") p.add_argument("--chat-id") p.add_argument("--owner-id") p.add_argument("--output", metavar="FILE") args = p.parse_args() report = direct(args) if args.mode == "direct" else over_http(args) text = json.dumps(report, ensure_ascii=False, indent=2) if args.output: Path(args.output).write_text(text + "\n") print(text) if __name__ == "__main__": main()