forked from mtusi2026/mtusi2026lab7
353 lines
12 KiB
Python
Executable File
353 lines
12 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
import re
|
|
import struct
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
import zlib
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
MAX_UPLOAD = 10 * 1024 * 1024
|
|
COLUMNS = (
|
|
"id",
|
|
"chat_id",
|
|
"owner_id",
|
|
"filename",
|
|
"content_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"width",
|
|
"height",
|
|
"object_key",
|
|
"status",
|
|
)
|
|
|
|
|
|
def chunk(tag, data):
|
|
return (
|
|
struct.pack(">I", len(data))
|
|
+ tag
|
|
+ data
|
|
+ struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF)
|
|
)
|
|
|
|
|
|
def png(width, height):
|
|
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
|
header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
|
return (
|
|
b"\x89PNG\r\n\x1a\n"
|
|
+ chunk(b"IHDR", header)
|
|
+ chunk(b"IDAT", zlib.compress(raw, 1))
|
|
+ chunk(b"IEND", b"")
|
|
)
|
|
|
|
|
|
def sized_png(target):
|
|
height = max(1, target // (1024 * 3 + 1))
|
|
body = png(1024, height)
|
|
while len(body) > min(target, MAX_UPLOAD) and height > 1:
|
|
height -= max(1, (len(body) - target) // 3073 + 1)
|
|
body = png(1024, height)
|
|
return body
|
|
|
|
|
|
def run(cmd, stdin=None):
|
|
out = subprocess.run(cmd, cwd=ROOT, input=stdin, capture_output=True, check=False)
|
|
if out.returncode:
|
|
raise SystemExit(
|
|
f"command failed: {' '.join(cmd[:6])}\n{out.stderr.decode(errors='replace')[-500:]}"
|
|
)
|
|
return out.stdout.decode()
|
|
|
|
|
|
def psql(sql, stdin=None):
|
|
user, db = os.environ.get("PGUSER", "messenger"), os.environ.get("PGDATABASE", "messenger")
|
|
psql = [
|
|
"docker",
|
|
"compose",
|
|
"exec",
|
|
"-T",
|
|
"postgres",
|
|
"psql",
|
|
"-v",
|
|
"ON_ERROR_STOP=1",
|
|
"-qAt",
|
|
"-F",
|
|
" ",
|
|
]
|
|
return run([*psql, "-U", user, "-d", db, "-c", sql], stdin)
|
|
|
|
|
|
def direct(args):
|
|
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
|
prefix = os.environ.get("S3_PREFIX", "attachments/")
|
|
if args.chat_id and args.owner_id:
|
|
chat_id, owner_id = args.chat_id, args.owner_id
|
|
else:
|
|
found = psql(
|
|
"SELECT chat_id, owner_id FROM attachments ORDER BY created_at LIMIT 1"
|
|
).split()
|
|
if len(found) != 2:
|
|
raise SystemExit(
|
|
"no attachment rows yet: upload one via API or pass --chat-id and --owner-id"
|
|
)
|
|
chat_id, owner_id = found
|
|
rows = []
|
|
started = time.monotonic()
|
|
with tempfile.TemporaryDirectory(prefix="lab4-seed-") as tmp:
|
|
Path(tmp).chmod(0o755)
|
|
for i in range(args.count):
|
|
aid = str(uuid.uuid4())
|
|
body = png(8, 8)
|
|
path = Path(tmp) / aid
|
|
path.write_bytes(body)
|
|
path.chmod(0o644)
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
rows.append(
|
|
f"{aid},{chat_id},{owner_id},seed-{i}.png,image/png,{len(body)},{digest},8,8,{prefix}{aid},ready"
|
|
)
|
|
generated = time.monotonic()
|
|
cli = [
|
|
"docker",
|
|
"compose",
|
|
"run",
|
|
"--rm",
|
|
"--no-deps",
|
|
"-T",
|
|
"-v",
|
|
f"{tmp}:/seed:ro",
|
|
"s3-cli",
|
|
]
|
|
run([*cli, "put", "--recursive", "--no-progress", "/seed/", f"root/{bucket}/{prefix}"])
|
|
uploaded = time.monotonic()
|
|
columns = ", ".join(COLUMNS)
|
|
psql(
|
|
f"COPY attachments ({columns}) FROM STDIN WITH (FORMAT csv)",
|
|
"".join(f"{r}\n" for r in rows).encode(),
|
|
)
|
|
seconds = {
|
|
"generate": round(generated - started, 3),
|
|
"s3_put": round(uploaded - generated, 3),
|
|
"pg_copy": round(time.monotonic() - uploaded, 3),
|
|
}
|
|
return {"mode": "direct", "count": args.count, "seconds": seconds}
|
|
|
|
|
|
class Http:
|
|
def __init__(self, base, timeout):
|
|
parts = urlsplit(base)
|
|
self.cls = (
|
|
http.client.HTTPSConnection if parts.scheme == "https" else http.client.HTTPConnection
|
|
)
|
|
self.host, self.timeout, self.local = parts.netloc, timeout, threading.local()
|
|
|
|
def call(self, method, path, body=None, headers=None, token=None):
|
|
conn = getattr(self.local, "conn", None) or self.cls(self.host, timeout=self.timeout)
|
|
self.local.conn = conn
|
|
h = {"Accept": "application/json"}
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
if isinstance(body, dict):
|
|
body, h["Content-Type"] = json.dumps(body).encode(), "application/json"
|
|
h.update(headers or {})
|
|
try:
|
|
conn.request(method, path, body=body, headers=h)
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
except (OSError, http.client.HTTPException):
|
|
conn.close()
|
|
self.local.conn = None
|
|
return 0, {}, b""
|
|
return response.status, {k.lower(): v for k, v in response.getheaders()}, data
|
|
|
|
|
|
def login(api, label):
|
|
name = f"seed_{label}_{uuid.uuid4().hex[:12]}"
|
|
password = "Seed-" + uuid.uuid4().hex
|
|
body = {"username": name, "display_name": label, "password": password}
|
|
status, _, data = api.call("POST", "/api/v1/users", body)
|
|
if status != 201:
|
|
raise SystemExit(f"user create failed: {status}")
|
|
user = json.loads(data)
|
|
basic = base64.b64encode(f"{name}:{password}".encode()).decode()
|
|
status, _, data = api.call(
|
|
"POST", "/api/v1/auth/sessions", headers={"Authorization": f"Basic {basic}"}
|
|
)
|
|
if status != 201:
|
|
raise SystemExit(f"login failed: {status}")
|
|
return user["id"], json.loads(data)["token"]
|
|
|
|
|
|
def mib(text):
|
|
m = re.match(r"([\d.]+)\s*([KMG]i?B)", text)
|
|
if not m:
|
|
return 0.0
|
|
scale = {"KiB": 1 / 1024, "KB": 1 / 1024, "GiB": 1024, "GB": 1024}
|
|
return float(m.group(1)) * scale.get(m.group(2), 1)
|
|
|
|
|
|
def sampler(ids, peaks, stop):
|
|
while not stop.is_set():
|
|
cmd = ["docker", "stats", "--no-stream", "--format", "{{.ID}} {{.MemUsage}}", *ids]
|
|
out = subprocess.run(cmd, capture_output=True, text=True, check=False).stdout
|
|
for line in out.splitlines():
|
|
cid, usage = line.split(" ", 1)
|
|
peaks[cid] = max(peaks.get(cid, 0.0), mib(usage))
|
|
|
|
|
|
def cgroup(cid, name):
|
|
cmd = ["docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}"]
|
|
return subprocess.run(cmd, capture_output=True, text=True, check=False).stdout.strip()
|
|
|
|
|
|
def percentile(values, q):
|
|
return (
|
|
round(sorted(values)[min(len(values) - 1, int(q * len(values)))] * 1000, 1)
|
|
if values
|
|
else None
|
|
)
|
|
|
|
|
|
def over_http(args):
|
|
api = Http(args.base_url, args.timeout)
|
|
alice, token = login(api, "alice")
|
|
bob, _ = login(api, "bob")
|
|
status, _, data = api.call(
|
|
"POST", "/api/v1/chats", {"title": "seed", "member_ids": [alice, bob]}, token=token
|
|
)
|
|
if status != 201:
|
|
raise SystemExit(f"chat create failed: {status}")
|
|
chat = json.loads(data)["id"]
|
|
fixed = sized_png(args.size) if args.size else None
|
|
counter, lock, results = iter(range(args.count)), threading.Lock(), []
|
|
ids = run(["docker", "compose", "ps", "-q", args.service]).split() if args.sample else []
|
|
idle = {cid: round(int(cgroup(cid, "memory.current") or 0) / 1048576, 1) for cid in ids}
|
|
peaks, stop = {}, threading.Event()
|
|
watcher = threading.Thread(target=sampler, args=(ids, peaks, stop), daemon=True)
|
|
if ids:
|
|
watcher.start()
|
|
|
|
def worker():
|
|
while True:
|
|
with lock:
|
|
i = next(counter, None)
|
|
if i is None:
|
|
return
|
|
body = fixed or png(8, 8)
|
|
began = time.monotonic()
|
|
path = f"/api/v1/chats/{chat}/attachments?filename=seed-{i}.png"
|
|
status, headers, data = api.call(
|
|
"POST", path, body, {"Content-Type": "image/png"}, token
|
|
)
|
|
created = json.loads(data)["id"] if status == 201 else None
|
|
results.append(
|
|
(status, time.monotonic() - began, headers.get("x-instance-id", ""), created)
|
|
)
|
|
|
|
started = time.monotonic()
|
|
threads = [threading.Thread(target=worker) for _ in range(args.concurrency)]
|
|
for t in threads:
|
|
t.start()
|
|
for t in threads:
|
|
t.join()
|
|
elapsed = time.monotonic() - started
|
|
stop.set()
|
|
if ids:
|
|
watcher.join()
|
|
created = [r[3] for r in results if r[3]]
|
|
for start in range(0, min(args.link, len(created)), 10):
|
|
body = {"text": "", "attachment_ids": created[start : min(start + 10, args.link)]}
|
|
api.call(
|
|
"POST",
|
|
f"/api/v1/chats/{chat}/messages",
|
|
body,
|
|
{"Idempotency-Key": uuid.uuid4().hex},
|
|
token,
|
|
)
|
|
if args.ids_out:
|
|
Path(args.ids_out).write_text("".join(f"{i}\n" for i in created))
|
|
codes, instances = {}, {}
|
|
for status, _, instance, _ in results:
|
|
codes[str(status)] = codes.get(str(status), 0) + 1
|
|
instances[instance] = instances.get(instance, 0) + 1
|
|
ok = [r[1] for r in results if r[0] == 201]
|
|
memory = {}
|
|
for cid in ids:
|
|
peak = cgroup(cid, "memory.peak")
|
|
events = dict(line.split() for line in cgroup(cid, "memory.events").splitlines())
|
|
memory[cid[:12]] = {
|
|
"idle_mib": idle[cid],
|
|
"docker_stats_peak_mib": round(peaks.get(cid[:12], 0.0), 1),
|
|
"cgroup_memory_peak_mib": round(int(peak) / 1048576, 1) if peak.isdigit() else None,
|
|
"oom_kill": int(events.get("oom_kill", 0)),
|
|
}
|
|
return {
|
|
"mode": "http",
|
|
"count": args.count,
|
|
"concurrency": args.concurrency,
|
|
"body_bytes": len(fixed) if fixed else "~200 (unique 8x8 PNG)",
|
|
"status_codes": codes,
|
|
"instances": len([i for i in instances if i]),
|
|
"linked_in_messages": min(args.link, len(created)),
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_rps": round(len(ok) / elapsed, 2) if elapsed else None,
|
|
"latency_ms": {
|
|
"p50": percentile(ok, 0.5),
|
|
"p95": percentile(ok, 0.95),
|
|
"p99": percentile(ok, 0.99),
|
|
"max": percentile(ok, 1.0),
|
|
},
|
|
"memory": memory,
|
|
}
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(
|
|
description="Lab4 attachment volume generator: HTTP uploads or direct S3+PG seeding"
|
|
)
|
|
p.add_argument("--mode", choices=["http", "direct"], default="http")
|
|
p.add_argument("--count", type=int, default=1000)
|
|
p.add_argument("--concurrency", type=int, default=16)
|
|
p.add_argument(
|
|
"--size",
|
|
type=int,
|
|
default=0,
|
|
help="http: bytes per upload (one PNG reused); 0 = unique 8x8 PNGs",
|
|
)
|
|
p.add_argument(
|
|
"--link", type=int, default=0, help="http: reference the first N attachments from messages"
|
|
)
|
|
p.add_argument(
|
|
"--sample",
|
|
action="store_true",
|
|
help="http: sample docker stats and memory.peak of API containers",
|
|
)
|
|
p.add_argument("--service", default="api")
|
|
p.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080"))
|
|
p.add_argument("--timeout", type=float, default=60)
|
|
p.add_argument("--ids-out", metavar="FILE")
|
|
p.add_argument("--chat-id")
|
|
p.add_argument("--owner-id")
|
|
p.add_argument("--output", metavar="FILE")
|
|
args = p.parse_args()
|
|
report = direct(args) if args.mode == "direct" else over_http(args)
|
|
text = json.dumps(report, ensure_ascii=False, indent=2)
|
|
if args.output:
|
|
Path(args.output).write_text(text + "\n")
|
|
print(text)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|