Балансировка, реплики, идемпотентность

This commit is contained in:
Queenfish
2026-10-16 01:07:34 +03:00
parent 0ab3a341aa
commit cb11011869
59 changed files with 3916 additions and 27 deletions
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env python3
import threading
import time
import uuid
from collections import Counter
import httpx
BASE = "http://localhost:8080"
def main():
counter = Counter()
lock = threading.Lock()
def worker():
with httpx.Client(timeout=5) as c:
for _ in range(20):
try:
r = c.get(f"{BASE}/health/live")
if r.status_code == 200:
with lock:
counter[r.headers.get("x-instance-id")] += 1
except Exception:
pass
time.sleep(0.05)
threads = [threading.Thread(target=worker) for _ in range(20)]
for t in threads:
t.start()
for t in threads:
t.join()
print(counter)
if __name__ == "__main__":
main()
+201
View File
@@ -0,0 +1,201 @@
#!/usr/bin/env python3
import base64
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx
BASE = "http://localhost:8080"
ROOT = Path(__file__).resolve().parent.parent
EVIDENCE = ROOT / "evidence" / "detection.json"
STOP_LABEL = "mtusi2026lab3-api-"
def register_and_login(client: httpx.Client) -> dict:
username = "det_" + uuid.uuid4().hex[:10]
password = "Test-password-" + uuid.uuid4().hex[:8]
r = client.post(
f"{BASE}/api/v1/users",
json={"username": username, "display_name": "D", "password": password},
)
assert r.status_code == 201, r.text
user = r.json()
credential = base64.b64encode(
(username + ":" + password).encode("utf-8")
).decode("ascii")
r = client.post(
f"{BASE}/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential},
)
assert r.status_code == 201, r.text
return {"id": user["id"], "token": r.json()["token"]}
def find_container_by_instance(target: str) -> str:
names = subprocess.run(
["docker", "ps", "--filter", f"name={STOP_LABEL}", "--format", "{{.Names}}"],
capture_output=True, text=True, check=True,
).stdout.split()
for name in names:
result = subprocess.run(
["docker", "logs", name],
capture_output=True, text=True, check=False,
)
combined = result.stdout + result.stderr
if f"INSTANCE_ID={target}" in combined:
return name
raise RuntimeError(f"no container with INSTANCE_ID={target}")
def get_instances(n: int = 30) -> set:
seen = set()
with httpx.Client(timeout=2) as c:
for _ in range(n):
try:
r = c.get(f"{BASE}/health/live")
if r.status_code == 200:
seen.add(r.headers.get("x-instance-id"))
except Exception:
pass
return seen
def main():
print("[measure] preparing user and chat", flush=True)
with httpx.Client(timeout=5) as c:
alice = register_and_login(c)
bob = register_and_login(c)
chat = c.post(
f"{BASE}/api/v1/chats",
json={"title": "det", "member_ids": [alice["id"], bob["id"]]},
headers={"Authorization": "Bearer " + alice["token"]},
).json()
instances_before = get_instances()
if len(instances_before) < 2:
print(f"[measure] need >=2 replicas, got {instances_before}", file=sys.stderr)
return 1
print(f"[measure] instances before: {instances_before}", flush=True)
target = sorted(instances_before)[0]
container = find_container_by_instance(target)
print(f"[measure] will stop {container} (instance={target})", flush=True)
shared = {
"latencies": [],
"errors": 0,
"instances": [],
"stop_time": None,
"first_success_after_stop": None,
"first_success_instance": None,
"stop_seen": False,
}
lock = threading.Lock()
running = {"flag": True}
def worker():
with httpx.Client(timeout=3) as c:
while running["flag"]:
t0 = time.monotonic()
try:
r = c.post(
f"{BASE}/api/v1/chats/{chat['id']}/messages",
json={"text": "det-" + uuid.uuid4().hex[:8]},
headers={
"Authorization": "Bearer " + alice["token"],
"Idempotency-Key": uuid.uuid4().hex,
},
)
dt = time.monotonic() - t0
if r.status_code == 201:
inst = r.headers.get("x-instance-id")
with lock:
shared["latencies"].append(dt)
shared["instances"].append(inst)
if shared["stop_time"] is not None and not shared["stop_seen"]:
if inst != target:
shared["first_success_after_stop"] = time.monotonic() - shared["stop_time"]
shared["first_success_instance"] = inst
shared["stop_seen"] = True
else:
with lock:
shared["errors"] += 1
except Exception:
with lock:
shared["errors"] += 1
print("[measure] starting background load", flush=True)
threads = [threading.Thread(target=worker) for _ in range(4)]
for t in threads:
t.start()
time.sleep(3)
with lock:
baseline_count = len(shared["latencies"])
baseline_rps = baseline_count / 3.0
print(f"[measure] baseline RPS ~{baseline_rps:.1f}", flush=True)
print(f"[measure] stopping {container}", flush=True)
with lock:
shared["stop_time"] = time.monotonic()
subprocess.run(["docker", "stop", "--time=2", container], check=True)
deadline = time.monotonic() + 20
while time.monotonic() < deadline:
with lock:
if shared["first_success_after_stop"] is not None:
break
time.sleep(0.05)
time.sleep(2)
running["flag"] = False
for t in threads:
t.join(timeout=5)
with lock:
report = {
"target_instance": target,
"target_container": container,
"instances_before": list(instances_before),
"total_successful": len(shared["latencies"]),
"total_errors": shared["errors"],
"baseline_rps_3s": round(baseline_rps, 2),
"detection_seconds": (
round(shared["first_success_after_stop"], 3)
if shared["first_success_after_stop"] else None
),
"first_success_instance_after_stop": shared["first_success_instance"],
}
print(json.dumps(report, ensure_ascii=False, indent=2))
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
existing = []
if EVIDENCE.exists():
try:
existing = json.loads(EVIDENCE.read_text(encoding="utf-8"))
except json.JSONDecodeError:
existing = []
existing.append(report)
EVIDENCE.write_text(
json.dumps(existing, indent=2, ensure_ascii=False),
encoding="utf-8",
)
print(f"[measure] restoring {container}", flush=True)
subprocess.run(
["docker", "compose", "--env-file", ".env", "up", "-d", "--scale", "api=2"],
cwd=ROOT, check=True,
)
time.sleep(5)
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -0,0 +1,2 @@
@echo off
python "%~dp0restart.py" %*
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
import subprocess
import sys
import time
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
BASE_URL = "http://localhost:8080"
def run(cmd):
print(f"[restart] $ {' '.join(cmd)}", flush=True)
result = subprocess.run(cmd, cwd=ROOT)
if result.returncode != 0:
print(
f"[restart] command failed with code {result.returncode}",
file=sys.stderr,
)
sys.exit(result.returncode)
def main():
run(["docker", "compose", "--env-file", ".env", "down"])
run([
"docker", "compose", "--env-file", ".env",
"up", "-d", "--build", "--scale", "api=2",
])
print("[restart] waiting for /health/ready", flush=True)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(
f"{BASE_URL}/health/ready", timeout=2
) as r:
if r.status == 200:
print("[restart] api is ready", flush=True)
return 0
except Exception:
time.sleep(1)
print(
"[restart] api did not become ready in 90 seconds",
file=sys.stderr,
)
return 1
if __name__ == "__main__":
sys.exit(main())
+22
View File
@@ -0,0 +1,22 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
echo "[restart.sh] docker compose down (volumes preserved)"
docker compose --env-file .env down
echo "[restart.sh] docker compose up -d --build --scale api=2"
docker compose --env-file .env up -d --build --scale api=2
echo "[restart.sh] waiting for /health/ready"
for i in $(seq 1 90); do
if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then
echo "[restart.sh] api is ready"
exit 0
fi
sleep 1
done
echo "[restart.sh] api did not become ready in 90 seconds" >&2
exit 1
+111
View File
@@ -0,0 +1,111 @@
#!/usr/bin/env python3
import base64
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx
BASE = "http://localhost:8080"
ROOT = Path(__file__).resolve().parent.parent
EVIDENCE = ROOT / "evidence" / "rolling-update.json"
def register_and_login(client):
username = "roll_" + uuid.uuid4().hex[:10]
password = "Test-password-" + uuid.uuid4().hex[:8]
r = client.post(
f"{BASE}/api/v1/users",
json={"username": username, "display_name": "R", "password": password},
)
assert r.status_code == 201, r.text
user = r.json()
credential = base64.b64encode((username + ":" + password).encode()).decode()
r = client.post(
f"{BASE}/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential},
)
assert r.status_code == 201, r.text
return {"id": user["id"], "token": r.json()["token"]}
def main():
with httpx.Client(timeout=5) as c:
alice = register_and_login(c)
bob = register_and_login(c)
chat = c.post(
f"{BASE}/api/v1/chats",
json={"title": "roll", "member_ids": [alice["id"], bob["id"]]},
headers={"Authorization": "Bearer " + alice["token"]},
).json()
running = {"flag": True}
latencies = []
errors = 0
lock = threading.Lock()
def worker():
nonlocal errors
with httpx.Client(timeout=3) as c:
while running["flag"]:
t0 = time.monotonic()
try:
r = c.post(
f"{BASE}/api/v1/chats/{chat['id']}/messages",
json={"text": "roll-" + uuid.uuid4().hex[:8]},
headers={
"Authorization": "Bearer " + alice["token"],
"Idempotency-Key": uuid.uuid4().hex,
},
)
dt = time.monotonic() - t0
if r.status_code == 201:
with lock:
latencies.append(dt)
else:
with lock:
errors += 1
except Exception:
with lock:
errors += 1
threads = [threading.Thread(target=worker) for _ in range(4)]
for t in threads:
t.start()
time.sleep(3)
with lock:
before = len(latencies)
print(f"[roll] baseline RPS ~{before/3:.1f}", flush=True)
for replica_name in ["mtusi2026lab3-api-1", "mtusi2026lab3-api-2"]:
print(f"[roll] restarting {replica_name}", flush=True)
subprocess.run(["docker", "restart", replica_name], check=True)
time.sleep(8)
time.sleep(2)
running["flag"] = False
for t in threads:
t.join(timeout=5)
with lock:
report = {
"total_successful": len(latencies),
"total_errors": errors,
"baseline_rps_3s": round(before / 3.0, 2),
"avg_latency_ms": round(1000 * sum(latencies) / len(latencies), 2) if latencies else None,
}
print(json.dumps(report, indent=2, ensure_ascii=False))
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())
+194
View File
@@ -0,0 +1,194 @@
#!/usr/bin/env python3
import base64
import json
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx
BASE = "http://localhost:8080"
ROOT = Path(__file__).resolve().parent.parent
EVIDENCE = ROOT / "evidence" / "scaling.json"
def register_and_login(client):
username = "scale_" + uuid.uuid4().hex[:10]
password = "Test-password-" + uuid.uuid4().hex[:8]
r = client.post(
f"{BASE}/api/v1/users",
json={"username": username, "display_name": "S", "password": password},
)
assert r.status_code == 201, r.text
user = r.json()
credential = base64.b64encode((username + ":" + password).encode()).decode()
r = client.post(
f"{BASE}/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential},
)
assert r.status_code == 201, r.text
return {"id": user["id"], "token": r.json()["token"]}
def run_load(chat_id: str, token: str, concurrency: int, duration: float):
latencies = []
errors = 0
lock = threading.Lock()
def worker():
nonlocal errors
local_lat = []
local_err = 0
with httpx.Client(timeout=15) as c:
deadline = time.monotonic() + duration
i = 0
while time.monotonic() < deadline:
t0 = time.monotonic()
try:
if i % 10 == 0:
r = c.get(
f"{BASE}/api/v1/chats/{chat_id}/messages?limit=10",
headers={"Authorization": "Bearer " + token},
)
else:
r = c.post(
f"{BASE}/api/v1/chats/{chat_id}/messages",
json={"text": "s-" + uuid.uuid4().hex[:6]},
headers={
"Authorization": "Bearer " + token,
"Idempotency-Key": uuid.uuid4().hex,
},
)
if r.status_code not in (200, 201):
local_err += 1
else:
local_lat.append(time.monotonic() - t0)
except Exception:
local_err += 1
i += 1
with lock:
latencies.extend(local_lat)
errors += local_err
threads = [threading.Thread(target=worker) for _ in range(concurrency)]
started = time.monotonic()
for t in threads:
t.start()
for t in threads:
t.join()
elapsed = time.monotonic() - started
latencies.sort()
def pct(p):
if not latencies:
return None
idx = max(0, int(len(latencies) * p) - 1)
return round(latencies[idx] * 1000, 2)
return {
"concurrency": concurrency,
"duration_s": round(elapsed, 2),
"success": len(latencies),
"errors": errors,
"rps": round(len(latencies) / elapsed, 2) if elapsed else 0,
"p50_ms": pct(0.50),
"p95_ms": pct(0.95),
"p99_ms": pct(0.99),
}
def switch_stack(replicas: int, cpus: str, memory: str):
env = dict(os.environ)
env["API_CPUS"] = cpus
env["API_MEMORY"] = memory
subprocess.run(
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml", "down"],
cwd=ROOT, check=True, env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
subprocess.run(
["docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", "compose-scale.yaml",
"up", "-d", "--build", "--scale", f"api={replicas}"],
cwd=ROOT, check=True, env=env,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
for _ in range(120):
try:
with httpx.Client(timeout=2) as c:
r = c.get(f"{BASE}/health/ready")
if r.status_code == 200:
time.sleep(8)
return
except Exception:
pass
time.sleep(1)
raise RuntimeError("API did not become ready")
def experiment_series(name, configurations, concurrency):
results = []
for cfg in configurations:
replicas = cfg["replicas"]
cpus = cfg["cpus"]
memory = cfg["memory"]
print(f"\n=== {name}: replicas={replicas}, cpus={cpus}, memory={memory}, clients={concurrency} ===", flush=True)
switch_stack(replicas, cpus, memory)
with httpx.Client(timeout=15) as c:
alice = register_and_login(c)
bob = register_and_login(c)
chat = c.post(
f"{BASE}/api/v1/chats",
json={"title": "scale", "member_ids": [alice["id"], bob["id"]]},
headers={"Authorization": "Bearer " + alice["token"]},
).json()
for run in range(3):
print(f" run {run+1}/3...", flush=True)
m = run_load(chat["id"], alice["token"], concurrency=concurrency, duration=20)
m["replicas"] = replicas
m["cpus_per_replica"] = cpus
m["memory_per_replica"] = memory
m["run"] = run + 1
m["series"] = name
results.append(m)
print(f" RPS={m['rps']}, p95={m['p95_ms']}, errors={m['errors']}", flush=True)
return results
def main():
all_results = []
print("### Baseline: 1 replica, 2 CPU, 20 clients", flush=True)
all_results.extend(experiment_series(
"baseline",
[{"replicas": 1, "cpus": "2.0", "memory": "512M"}],
concurrency=20,
))
print("\n### Improved: 2 replicas, 1 CPU, 20 clients", flush=True)
all_results.extend(experiment_series(
"improved",
[{"replicas": 2, "cpus": "1.0", "memory": "256M"}],
concurrency=20,
))
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
EVIDENCE.write_text(
json.dumps(all_results, indent=2, ensure_ascii=False),
encoding="utf-8",
)
print(f"\nSaved {len(all_results)} records to {EVIDENCE}")
return 0
if __name__ == "__main__":
sys.exit(main())
+2
View File
@@ -0,0 +1,2 @@
@echo off
python "%~dp0stop-one.py" %*
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env python3
import os
import subprocess
import sys
def main():
target = os.environ.get("TARGET_INSTANCE_ID")
if not target:
print("[stop-one] TARGET_INSTANCE_ID not set", file=sys.stderr)
return 1
names = subprocess.run(
["docker", "ps", "--filter", "name=mtusi2026lab3-api-", "--format", "{{.Names}}"],
capture_output=True, text=True, check=True,
).stdout.split()
for container in names:
logs = subprocess.run(
["docker", "logs", container],
capture_output=True, text=True, check=False,
)
combined = logs.stdout + logs.stderr
if f"INSTANCE_ID={target}" in combined:
print(f"[stop-one] stopping {container} (instance={target})", flush=True)
subprocess.run(["docker", "stop", "--time=2", container], check=True)
return 0
print(f"[stop-one] no container found for INSTANCE_ID={target}", file=sys.stderr)
return 1
if __name__ == "__main__":
sys.exit(main())
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
set -euo pipefail
TARGET="${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID not set}"
for container in $(docker ps --filter "name=mtusi2026lab3-api-" --format "{{.Names}}"); do
if docker logs "$container" 2>&1 | grep -q "INSTANCE_ID=$TARGET"; then
echo "[stop-one] stopping $container (instance=$TARGET)"
docker stop "$container"
exit 0
fi
done
echo "[stop-one] no container found for INSTANCE_ID=$TARGET" >&2
exit 1
+105
View File
@@ -0,0 +1,105 @@
#!/usr/bin/env python3
import base64
import json
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx
BASE = "http://localhost:8080"
ROOT = Path(__file__).resolve().parent.parent
EVIDENCE = ROOT / "evidence" / "idempotency.json"
def register_and_login(client):
username = "idem_" + uuid.uuid4().hex[:10]
password = "Test-password-" + uuid.uuid4().hex[:8]
r = client.post(
f"{BASE}/api/v1/users",
json={"username": username, "display_name": "I", "password": password},
)
assert r.status_code == 201, r.text
user = r.json()
credential = base64.b64encode((username + ":" + password).encode()).decode()
r = client.post(
f"{BASE}/api/v1/auth/sessions",
headers={"Authorization": "Basic " + credential},
)
assert r.status_code == 201, r.text
return {"id": user["id"], "token": r.json()["token"]}
def main():
with httpx.Client(timeout=10) as c:
alice = register_and_login(c)
bob = register_and_login(c)
chat = c.post(
f"{BASE}/api/v1/chats",
json={"title": "idem", "member_ids": [alice["id"], bob["id"]]},
headers={"Authorization": "Bearer " + alice["token"]},
).json()
key = uuid.uuid4().hex
text = "idempotency-test-" + uuid.uuid4().hex[:8]
results = []
lock = threading.Lock()
barrier = threading.Barrier(10)
def worker():
try:
with httpx.Client(timeout=10) as c:
barrier.wait()
r = c.post(
f"{BASE}/api/v1/chats/{chat['id']}/messages",
json={"text": text},
headers={
"Authorization": "Bearer " + alice["token"],
"Idempotency-Key": key,
},
)
with lock:
results.append((r.status_code, r.headers.get("x-instance-id")))
except Exception as e:
with lock:
results.append((0, str(e)))
threads = [threading.Thread(target=worker) for _ in range(10)]
for t in threads:
t.start()
for t in threads:
t.join()
time.sleep(1)
with httpx.Client(timeout=10) as c:
page = c.get(
f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=100",
headers={"Authorization": "Bearer " + alice["token"]},
).json()
matching = [m for m in page["items"] if m["text"] == text]
message_ids = {m["id"] for m in matching}
instances_seen = list({r[1] for r in results if r[1] and r[0] > 0})
statuses = [r[0] for r in results]
report = {
"concurrent_requests": len(results),
"statuses": statuses,
"unique_instances": instances_seen,
"messages_created": len(matching),
"unique_message_ids": len(message_ids),
"idempotency_ok": len(matching) == 1,
"instances_count": len(instances_seen),
}
print(json.dumps(report, indent=2, ensure_ascii=False))
EVIDENCE.parent.mkdir(parents=True, exist_ok=True)
EVIDENCE.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
return 0
if __name__ == "__main__":
sys.exit(main())