"""Генератор нагрузки для экспериментов lab4+ (asyncio + aiohttp, closed-loop). Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %). При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых сообщений (для сверки с историей), число повторов. """ import asyncio import base64 import random import time import uuid import aiohttp RETRIES = 3 class Stand: """Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента.""" def __init__(self, base, users=20, chats=20): self.base, self.n_users, self.n_chats = base, users, chats self.users, self.chats = [], [] async def setup(self): timeout = aiohttp.ClientTimeout(total=30) async with aiohttp.ClientSession(timeout=timeout) as s: for i in range(self.n_users): name = "ld_%s" % uuid.uuid4().hex[:14] pw = "load-test-password-" + uuid.uuid4().hex[:8] async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r: user = await r.json() assert r.status == 201, user cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: sess = await r.json() assert r.status == 201, sess self.users.append({"id": user["id"], "token": sess["token"]}) for i in range(self.n_chats): members = random.sample(self.users, 3) owner = members[0] async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]}, headers={"Authorization": "Bearer " + owner["token"]}) as r: chat = await r.json() assert r.status == 201, chat self.chats.append({"id": chat["id"], "members": members}) class Recorder: def __init__(self): self.t0 = time.time() self.events = [] # (t, kind, status, latency_ms, instance) self.confirmed = [] # (chat_id, message_id, text) self.retries = 0 self.final_failures = 0 self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!) async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0): connector = aiohttp.TCPConnector(limit=2) timeout = aiohttp.ClientTimeout(total=5) async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s: n = 0 while not deadline_fn(): chat = random.choice(stand.chats) user = random.choice(chat["members"]) hdr = {"Authorization": "Bearer " + user["token"]} if random.random() < read_ratio: t = time.perf_counter() try: async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r: await r.read() rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id"))) except Exception as exc: rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None)) continue key = uuid.uuid4().hex text = "m-%s-%d" % (key[:8], n) + "я" * text_size n += 1 for attempt in range(RETRIES + 1): t = time.perf_counter() try: async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text}, headers=dict(hdr, **{"Idempotency-Key": key})) as r: body = await r.json(content_type=None) status, inst = r.status, r.headers.get("X-Instance-Id") except Exception as exc: status, inst, body = type(exc).__name__, None, None rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst)) if status == 201: rec.confirmed.append((chat["id"], body["id"], text)) break if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429): rec.retries += 1 await asyncio.sleep(0.05 * (attempt + 1)) continue rec.final_failures += 1 rec.unconfirmed.append((chat["id"], key, text)) break if think: await asyncio.sleep(think) async def run_load(stand, rec, seconds, concurrency, **kw): end = time.time() + seconds await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency))) async def verify(stand, rec): """Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза.""" timeout = aiohttp.ClientTimeout(total=30) stored = {} async with aiohttp.ClientSession(timeout=timeout) as s: for chat in stand.chats: hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]} cursor, items = None, [] while True: url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "") async with s.get(url, headers=hdr) as r: page = await r.json() assert r.status == 200, page items += page["items"] cursor = page["next_cursor"] if not cursor: break stored[chat["id"]] = items ids = {m["id"] for items in stored.values() for m in items} texts = [m["text"] for items in stored.values() for m in items] confirmed_missing = [c for c in rec.confirmed if c[1] not in ids] dup_texts = len(texts) - len(set(texts)) unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts)) return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing), "duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed), "unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures} def per_second(rec, kind="POST"): buckets = {} for t, k, status, lat, inst in rec.events: if k != kind: continue b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()}) if status == 201 or (kind == "GET" and status == 200): b["ok"] += 1 b["lat"].append(lat) if inst: b["instances"].add(inst) else: b["err"] += 1 return buckets def pct(values, p): if not values: return None values = sorted(values) return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2) def summarize(rec, t_from=None, t_to=None, kind="POST"): rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)] if not rows: return {} dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows)) ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)] errs = {} for e in rows: if not (e[2] == 201 or (kind == "GET" and e[2] == 200)): errs[str(e[2])] = errs.get(str(e[2]), 0) + 1 return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None, "error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)}