Очереди и асинхронная обработка изображений, трек 1

This commit is contained in:
Queenfish
2026-11-12 21:14:30 +03:00
parent 276dbdad8a
commit 01a5023c6d
102 changed files with 6374 additions and 47 deletions
+122
View File
@@ -0,0 +1,122 @@
import base64
import io
import os
import subprocess
import time
import uuid
import threading
import statistics
import httpx
from PIL import Image
BASE = "http://localhost:8080"
CONCURRENCY = 10
TOTAL_FILES = 30
def _png(size=(320, 200), color=(0, 255, 0)):
buf = io.BytesIO()
Image.new("RGB", size, color).save(buf, format="PNG")
return buf.getvalue()
def _user(c):
u = "bench_" + uuid.uuid4().hex[:8]
p = "Test-password-1234"
c.post(f"{BASE}/api/v1/users", json={"username": u, "display_name": "B", "password": p})
cred = base64.b64encode(f"{u}:{p}".encode()).decode()
s = c.post(f"{BASE}/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}).json()
return {"id": s["user"]["id"], "token": s["token"]}
def _chat(c, a, b):
return c.post(
f"{BASE}/api/v1/chats",
json={"title": "bench", "member_ids": [a["id"], b["id"]]},
headers={"Authorization": "Bearer " + a["token"]},
).json()
def _mem(container):
out = subprocess.run(
["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container],
capture_output=True, text=True, check=True,
).stdout.strip()
v = out.split("/")[0].strip()
units = {"B": 1, "KiB": 1024, "Ki": 1024, "MiB": 1024 ** 2, "Mi": 1024 ** 2, "GiB": 1024 ** 3, "Gi": 1024 ** 3}
for u in sorted(units, key=len, reverse=True):
if v.endswith(u):
return int(float(v[: -len(u)]) * units[u])
return 0
def main():
prefetch = os.getenv("PREFETCH_COUNT", "1")
data = _png()
with httpx.Client(timeout=30) as c:
a = _user(c)
b = _user(c)
chat = _chat(c, a, b)
containers = subprocess.run(
["docker", "ps", "--filter", "name=mtusi2026lab5-worker-", "--format", "{{.Names}}"],
capture_output=True, text=True, check=True,
).stdout.split()
if not containers:
raise SystemExit("worker not found")
worker_container = containers[0]
start_time = time.monotonic()
ids = []
def upload_one():
with httpx.Client(timeout=30) as c:
r = c.post(
f"{BASE}/api/v1/chats/{chat['id']}/attachments",
params={"filename": "bench.png"},
content=data,
headers={"Authorization": "Bearer " + a["token"], "Content-Type": "image/png"},
)
if r.status_code == 202:
return r.json()["id"]
return None
threads = []
for _ in range(TOTAL_FILES):
t = threading.Thread(target=lambda: ids.append(upload_one()))
t.start()
threads.append(t)
if len(threads) >= CONCURRENCY:
for t in threads: t.join()
threads = []
for t in threads: t.join()
upload_time = time.monotonic() - start_time
ids = [i for i in ids if i]
print(f"Uploaded {len(ids)} files in {upload_time:.2f}s. RPS (upload): {len(ids) / upload_time:.2f}")
ready_times = []
start_wait = time.monotonic()
for att_id in ids:
with httpx.Client(timeout=10) as c:
while True:
r = c.get(f"{BASE}/api/v1/attachments/{att_id}", headers={"Authorization": "Bearer " + a["token"]})
if r.status_code == 200 and r.json()["status"] == "ready":
ready_times.append(time.monotonic() - start_wait)
break
time.sleep(0.1)
total_time = time.monotonic() - start_time
p95 = statistics.quantiles(ready_times, n=20)[18] if len(ready_times) > 1 else 0
ram = _mem(worker_container)
print(f"--- RESULTS (prefetch={prefetch}) ---")
print(f"Total time: {total_time:.2f}s")
print(f"Throughput: {len(ids) / total_time:.2f} files/sec")
print(f"p95 time to ready: {p95:.2f}s")
print(f"Worker RAM: {ram / 1024 ** 2:.1f} MiB")
if __name__ == "__main__":
main()