forked from mtusi2026/mtusi2026lab7
Перенесён минимум lab6 (API без изменений), добавлена защита на границах доверия: - внешний HTTPS; обязательный mTLS LB -> API с allowlist identity и отзывом по serial; - TLS на всех связях: PostgreSQL verify-full, MinIO HTTPS, RabbitMQ AMQPS mTLS (EXTERNAL), Valkey TLS + ACL, Collector и хранилища телеметрии через mTLS с отдельными CA; - OpenBao в server mode: cert auth, токены 15 мин, политики только на свой путь; - роли БД, S3-политики и права брокера по принципу минимальных прав; - Argon2id с замером, rehash, общий лимит входа, бюджет защиты от login DoS; - hardening всех контейнеров, scan образов с разбором, модель угроз, матрица доверия; - отрицательные тесты: TLS-аудит (135 проверок), ACL, ротации, sealed OpenBao.
212 lines
9.4 KiB
Python
212 lines
9.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Open-loop load generator (stdlib only): fixed arrival rate, latency measured from the SCHEDULED send time,
|
|
so a slow server cannot slow down the generator (no coordinated omission while threads are available).
|
|
|
|
Mix per arrival: 75% POST message, 25% GET history (limit=20); plus an independent upload stream (sample.png).
|
|
Rare injected requests (FAULT_INJECTION=1 on the stand): `X-Fault: slow` (+600 ms) and `X-Fault: error` (500)
|
|
are added to GET history with the given percentages, to measure whether such traces survive sampling.
|
|
|
|
python3 student-tests/openloop.py --rate 40 --duration 60 --upload-rate 1 --slow-pct 1 --error-pct 1 \
|
|
--output evidence/x.json
|
|
Tokens stay in memory; the summary contains counts, percentiles and timestamps only.
|
|
"""
|
|
import argparse
|
|
import http.client
|
|
import json
|
|
import os
|
|
import queue
|
|
import random
|
|
import ssl
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
from urllib.parse import urlsplit
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "tests"))
|
|
os.environ.setdefault("CA_FILE", str(ROOT / "secrets/pki/ca.crt"))
|
|
from client import Client # noqa: E402
|
|
|
|
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
|
|
|
|
|
def percentile(values, p):
|
|
if not values:
|
|
return None
|
|
values = sorted(values)
|
|
k = max(0, min(len(values) - 1, int(round(p / 100.0 * len(values) + 0.5)) - 1))
|
|
return values[k]
|
|
|
|
|
|
class Load:
|
|
def __init__(self, base_url, users=4, threads=64, timeout=30.0):
|
|
self.base = urlsplit(base_url)
|
|
self.tls = ssl.create_default_context(cafile=os.environ.get("CA_FILE") or str(ROOT / "secrets/pki/ca.crt"))
|
|
self.threads = threads
|
|
self.timeout = timeout
|
|
c = Client(base_url)
|
|
self.actors = []
|
|
for i in range(users):
|
|
a, b = c.user("load%da" % i), c.user("load%db" % i)
|
|
chat = c.call("POST", "/api/v1/chats", 201, actor=a,
|
|
body={"title": "load %d" % i, "member_ids": [a["id"], b["id"]]}, schema="Chat")["json"]
|
|
self.actors.append((a["token"], chat["id"]))
|
|
self.lock = threading.Lock()
|
|
self.results = []
|
|
self.local = threading.local()
|
|
|
|
def conn(self):
|
|
c = getattr(self.local, "conn", None)
|
|
if c is None:
|
|
if self.base.scheme == "https": # lab7: TLS verified against the stand CA, never unverified
|
|
c = http.client.HTTPSConnection(self.base.hostname, self.base.port or 443, timeout=self.timeout,
|
|
context=self.tls)
|
|
else:
|
|
c = http.client.HTTPConnection(self.base.hostname, self.base.port or 80, timeout=self.timeout)
|
|
self.local.conn = c
|
|
return c
|
|
|
|
def request(self, method, path, body=None, headers=None):
|
|
for attempt in (1, 2): # second attempt only for a dropped keep-alive connection BEFORE sending
|
|
conn = self.conn()
|
|
try:
|
|
conn.request(method, path, body=body, headers=headers or {})
|
|
response = conn.getresponse()
|
|
response.read()
|
|
return response.status, response.getheader("X-Request-Id")
|
|
except (http.client.RemoteDisconnected, ConnectionResetError, BrokenPipeError) as exc:
|
|
conn.close()
|
|
self.local.conn = None
|
|
if attempt == 2:
|
|
return "conn_error:" + type(exc).__name__, None
|
|
except (OSError, http.client.HTTPException) as exc:
|
|
conn.close()
|
|
self.local.conn = None
|
|
return "error:" + type(exc).__name__, None
|
|
return "error", None
|
|
|
|
def one(self, kind, fault, scheduled):
|
|
token, chat = random.choice(self.actors)
|
|
headers = {"Authorization": "Bearer " + token}
|
|
if fault:
|
|
headers["X-Fault"] = fault
|
|
if kind == "post_message":
|
|
headers.update({"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex})
|
|
status, rid = self.request("POST", "/api/v1/chats/%s/messages" % chat,
|
|
json.dumps({"text": "нагрузка " + uuid.uuid4().hex[:8]}).encode(), headers)
|
|
elif kind == "get_history":
|
|
status, rid = self.request("GET", "/api/v1/chats/%s/messages?limit=20" % chat, None, headers)
|
|
else:
|
|
headers["Content-Type"] = "image/png"
|
|
status, rid = self.request("POST", "/api/v1/chats/%s/attachments?filename=load.png" % chat, SAMPLE, headers)
|
|
done = time.monotonic()
|
|
with self.lock:
|
|
self.results.append({"kind": kind, "fault": fault, "status": status, "latency": done - scheduled,
|
|
"at": time.time(), "rid": rid})
|
|
|
|
def run(self, rate, duration, upload_rate=0.0, slow_pct=0.0, error_pct=0.0, stop_event=None):
|
|
work = queue.Queue()
|
|
start = time.monotonic()
|
|
self.started_at = time.time()
|
|
|
|
def worker():
|
|
while True:
|
|
item = work.get()
|
|
if item is None:
|
|
return
|
|
self.one(*item)
|
|
|
|
pool = [threading.Thread(target=worker, daemon=True) for _ in range(self.threads)]
|
|
for t in pool:
|
|
t.start()
|
|
|
|
def schedule(r, picker):
|
|
if r <= 0:
|
|
return
|
|
n = 0
|
|
while True:
|
|
due = start + n / r
|
|
if due - start >= duration or (stop_event and stop_event.is_set()):
|
|
return
|
|
delay = due - time.monotonic()
|
|
if delay > 0:
|
|
time.sleep(delay)
|
|
work.put(picker() + (due,))
|
|
n += 1
|
|
|
|
def pick_main():
|
|
if random.random() < 0.75:
|
|
return ("post_message", None)
|
|
roll = random.random() * 100
|
|
fault = "slow" if roll < slow_pct * 4 else "error" if roll < (slow_pct + error_pct) * 4 else None
|
|
return ("get_history", fault) # 25% share * 4 = requested share of ALL requests
|
|
|
|
schedulers = [threading.Thread(target=schedule, args=(rate, pick_main), daemon=True),
|
|
threading.Thread(target=schedule, args=(upload_rate, lambda: ("upload", None)), daemon=True)]
|
|
for t in schedulers:
|
|
t.start()
|
|
for t in schedulers:
|
|
t.join()
|
|
backlog = work.qsize()
|
|
for _ in pool:
|
|
work.put(None)
|
|
for t in pool:
|
|
t.join(self.timeout + 5)
|
|
self.finished_at = time.time()
|
|
return self.summary(rate, duration, upload_rate, slow_pct, error_pct, backlog)
|
|
|
|
def summary(self, rate, duration, upload_rate, slow_pct, error_pct, backlog):
|
|
res = self.results
|
|
ok = [r for r in res if isinstance(r["status"], int) and r["status"] < 400]
|
|
plain_ok = [r["latency"] for r in ok if r["fault"] is None and r["kind"] != "upload"]
|
|
out = {
|
|
"profile": {"rate_rps": rate, "duration_s": duration, "upload_rate_rps": upload_rate,
|
|
"slow_pct": slow_pct, "error_pct": error_pct, "threads": self.threads,
|
|
"mix": "75% POST message / 25% GET history(limit=20); uploads separate"},
|
|
"started_at": self.started_at, "finished_at": self.finished_at, "generator_backlog_at_end": backlog,
|
|
"requests": len(res), "ok": len(ok),
|
|
"successful_rps": round(len(ok) / max(duration, 1e-9), 2),
|
|
"status_counts": {},
|
|
"server_errors_unexpected": sum(1 for r in res if r["fault"] != "error" and (
|
|
not isinstance(r["status"], int) or r["status"] >= 500)),
|
|
"latency_ms_plain_ok": {p: round(percentile(plain_ok, q) * 1000, 2) if plain_ok else None
|
|
for p, q in (("p50", 50), ("p95", 95), ("p99", 99), ("max", 100))},
|
|
"faults_sent": {"slow": sum(1 for r in res if r["fault"] == "slow"),
|
|
"error": sum(1 for r in res if r["fault"] == "error")},
|
|
"by_kind": {},
|
|
}
|
|
for r in res:
|
|
key = str(r["status"])
|
|
out["status_counts"][key] = out["status_counts"].get(key, 0) + 1
|
|
for kind in ("post_message", "get_history", "upload"):
|
|
lat = [r["latency"] for r in ok if r["kind"] == kind and r["fault"] is None]
|
|
out["by_kind"][kind] = {"n": sum(1 for r in res if r["kind"] == kind),
|
|
"p50_ms": round(percentile(lat, 50) * 1000, 2) if lat else None,
|
|
"p99_ms": round(percentile(lat, 99) * 1000, 2) if lat else None}
|
|
return out
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
p.add_argument("--base-url", default="https://localhost:8443")
|
|
p.add_argument("--rate", type=float, default=40)
|
|
p.add_argument("--duration", type=float, default=60)
|
|
p.add_argument("--upload-rate", type=float, default=0)
|
|
p.add_argument("--slow-pct", type=float, default=0)
|
|
p.add_argument("--error-pct", type=float, default=0)
|
|
p.add_argument("--threads", type=int, default=64)
|
|
p.add_argument("--output")
|
|
a = p.parse_args()
|
|
load = Load(a.base_url, threads=a.threads)
|
|
summary = load.run(a.rate, a.duration, a.upload_rate, a.slow_pct, a.error_pct)
|
|
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
|
print(text)
|
|
if a.output:
|
|
Path(a.output).write_text(text + "\n", encoding="utf-8")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|