forked from mtusi2026/mtusi2026lab4
Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок
This commit is contained in:
@@ -0,0 +1,245 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка записью через балансировщик во время отказа или замены реплик.
|
||||
|
||||
stop — по очереди останавливает каждую реплику (SIGTERM), восстанавливая предыдущую;
|
||||
kill — то же, но SIGKILL, без корректного завершения;
|
||||
pause — то же, но реплика зависает (docker pause): соединения открыты, ответов нет;
|
||||
rolling — запускает scripts/rolling-update.sh.
|
||||
|
||||
Клиент ведёт себя как в контракте: при неопределённом ответе повторяет запрос с тем же ключом.
|
||||
В конце вся история сверяется с подтверждёнными сообщениями.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
from api import call, history, new_chat, new_user, send
|
||||
|
||||
WORKERS = 4
|
||||
|
||||
|
||||
def compose(*args):
|
||||
command = ["docker", "compose", "--env-file", ".env", *args]
|
||||
return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def api_containers():
|
||||
return compose("ps", "-q", "api").split()
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if call("GET", "/health/ready", timeout=2)[0] == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
raise SystemExit("стенд не стал готов")
|
||||
|
||||
|
||||
def wait_replicas(count):
|
||||
"""Ждёт, пока балансировщик начнёт отдавать запросы на count разных реплик."""
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
seen = {call("GET", "/health/live", timeout=2)[2]["X-Instance-Id"] for _ in range(count * 4)}
|
||||
if len(seen) >= count:
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise SystemExit("балансировщик не видит %d реплик" % count)
|
||||
|
||||
|
||||
class Load:
|
||||
"""Несколько клиентов пишут в один чат, каждое событие сохраняется с отметкой времени."""
|
||||
|
||||
def __init__(self):
|
||||
self.user = new_user("chaos")
|
||||
self.chat = new_chat(self.user)
|
||||
self.stop = threading.Event()
|
||||
self.events = [] # (время, ok, реплика, статус)
|
||||
self.confirmed = {}
|
||||
self.lock = threading.Lock()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,)) for n in range(WORKERS)]
|
||||
|
||||
def worker(self, number):
|
||||
index = 0
|
||||
while not self.stop.is_set():
|
||||
text = "worker-%d-%d-%s" % (number, index, uuid.uuid4().hex[:8])
|
||||
key = uuid.uuid4().hex
|
||||
index += 1
|
||||
while True:
|
||||
try:
|
||||
status, body, headers = send(self.user, self.chat, text, key=key, timeout=3)
|
||||
instance = headers.get("X-Instance-Id")
|
||||
except OSError:
|
||||
status, body, instance = 0, None, None
|
||||
now = time.monotonic()
|
||||
ok = status == 201
|
||||
with self.lock:
|
||||
self.events.append((now, ok, instance, status))
|
||||
if ok:
|
||||
self.confirmed[body["id"]] = body
|
||||
if ok or self.stop.is_set():
|
||||
break
|
||||
time.sleep(0.05) # неопределённый результат: тот же текст и тот же ключ
|
||||
|
||||
def start(self):
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
for thread in self.threads:
|
||||
thread.join()
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
ordered = sorted(values)
|
||||
return round(ordered[max(0, math.ceil(p * len(ordered)) - 1)], 3) if ordered else None
|
||||
|
||||
|
||||
def summary(events, started, action_at, target=None):
|
||||
"""Показатели одного интервала: от action_at считается время до восстановления записи."""
|
||||
ok = [e for e in events if e[1]]
|
||||
failed = [e for e in events if not e[1]]
|
||||
elapsed = events[-1][0] - started
|
||||
gaps = [b[0] - a[0] for a, b in zip(ok, ok[1:])]
|
||||
after = [e for e in ok if e[0] >= action_at]
|
||||
last_on_target = [e[0] for e in ok if target and e[2] and target.startswith(e[2])]
|
||||
return {
|
||||
"seconds": round(elapsed, 2),
|
||||
"successful_requests": len(ok),
|
||||
"failed_attempts": len(failed),
|
||||
"failed_statuses": sorted({e[3] for e in failed}),
|
||||
"successful_rps": round(len(ok) / elapsed, 1),
|
||||
"longest_gap_between_successes_s": round(max(gaps), 3) if gaps else None,
|
||||
"first_success_after_action_s": round(after[0][0] - action_at, 3) if after else None,
|
||||
"last_success_on_target_after_action_s": round(last_on_target[-1] - action_at, 3)
|
||||
if last_on_target and last_on_target[-1] >= action_at else None,
|
||||
"instances_seen": len({e[2] for e in ok}),
|
||||
}
|
||||
|
||||
|
||||
def verify(load):
|
||||
items, _ = history(load.user, load.chat, 100)
|
||||
by_id = {item["id"]: item for item in items}
|
||||
texts = [item["text"] for item in items]
|
||||
lost = [mid for mid, message in load.confirmed.items() if by_id.get(mid) != message]
|
||||
result = {
|
||||
"confirmed": len(load.confirmed),
|
||||
"in_history": len(items),
|
||||
"lost_confirmed": len(lost),
|
||||
"duplicate_ids": len(items) - len(by_id),
|
||||
"duplicate_texts": len(texts) - len(set(texts)),
|
||||
}
|
||||
result["unconfirmed_but_saved"] = len(items) - len(load.confirmed)
|
||||
return result
|
||||
|
||||
|
||||
def replicas_up():
|
||||
"""Сколько реплик балансировщик считает рабочими, по его странице статистики."""
|
||||
csv = compose("exec", "-T", "lb", "wget", "-q", "-O", "-", "http://127.0.0.1:8404/stats;csv")
|
||||
return sum(line.startswith("api,api") and ",UP" in line for line in csv.splitlines())
|
||||
|
||||
|
||||
def seconds_until_excluded(replicas, since, limit=8):
|
||||
while time.monotonic() - since < limit:
|
||||
if replicas_up() < replicas:
|
||||
return round(time.monotonic() - since, 2)
|
||||
time.sleep(0.1)
|
||||
return None
|
||||
|
||||
|
||||
def replica_round(signal):
|
||||
containers = api_containers()
|
||||
wait_replicas(len(containers))
|
||||
rounds = []
|
||||
for container in containers:
|
||||
load = Load()
|
||||
started = time.monotonic()
|
||||
load.start()
|
||||
time.sleep(5)
|
||||
action_at = time.monotonic()
|
||||
command = {"SIGTERM": "stop", "SIGKILL": "kill", "HANG": "pause"}[signal]
|
||||
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
||||
stop_took = time.monotonic() - action_at
|
||||
excluded_after = seconds_until_excluded(len(containers), action_at)
|
||||
time.sleep(max(0, 10 - (time.monotonic() - action_at)))
|
||||
command = "unpause" if signal == "HANG" else "start"
|
||||
subprocess.run(["docker", command, container], check=True, stdout=subprocess.DEVNULL)
|
||||
wait_replicas(len(containers))
|
||||
time.sleep(2)
|
||||
load.finish()
|
||||
result = summary(load.events, started, action_at, container)
|
||||
result.update({"target": container[:12], "signal": signal, "stop_command_s": round(stop_took, 2),
|
||||
"excluded_by_lb_after_s": excluded_after})
|
||||
result["verify"] = verify(load)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
rounds.append(result)
|
||||
return rounds
|
||||
|
||||
|
||||
def images():
|
||||
inspect = ["docker", "inspect", "-f", "{{.Image}}"]
|
||||
return {c[:12]: subprocess.check_output(inspect + [c], text=True).strip()[:19] for c in api_containers()}
|
||||
|
||||
|
||||
def rolling_round():
|
||||
before = images()
|
||||
wait_replicas(len(before))
|
||||
load = Load()
|
||||
started = time.monotonic()
|
||||
load.start()
|
||||
time.sleep(5)
|
||||
action_at = time.monotonic()
|
||||
subprocess.run(["./scripts/rolling-update.sh"], cwd=ROOT, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
update_took = time.monotonic() - action_at
|
||||
time.sleep(5)
|
||||
load.finish()
|
||||
after = images()
|
||||
result = summary(load.events, started, action_at)
|
||||
result.update({"update_seconds": round(update_took, 2), "containers_before": before, "containers_after": after,
|
||||
"app_version": os.environ.get("APP_VERSION", "dev")})
|
||||
result["verify"] = verify(load)
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
# без этой проверки скрипт, который ничего не заменил, выглядел бы как идеальное обновление
|
||||
if set(before) & set(after) or set(before.values()) & set(after.values()) or len(after) != len(before):
|
||||
raise SystemExit("FAIL: реплики не заменены новым образом")
|
||||
return [result]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("mode", choices=["stop", "kill", "pause", "rolling"])
|
||||
parser.add_argument("--output")
|
||||
args = parser.parse_args()
|
||||
wait_ready()
|
||||
if args.mode == "rolling":
|
||||
rounds = rolling_round()
|
||||
else:
|
||||
rounds = replica_round({"stop": "SIGTERM", "kill": "SIGKILL", "pause": "HANG"}[args.mode])
|
||||
report = {
|
||||
"workload": "%d клиента, closed loop, POST сообщений через балансировщик, повтор с тем же ключом" % WORKERS,
|
||||
"mode": args.mode,
|
||||
"rounds": rounds,
|
||||
}
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
bad = [r for r in rounds if r["verify"]["lost_confirmed"] or r["verify"]["duplicate_ids"] or r["verify"]["duplicate_texts"]]
|
||||
if bad:
|
||||
raise SystemExit("FAIL: потеряны или продублированы сообщения")
|
||||
print("PASS: подтверждённые сообщения на месте, дублей нет")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user