This commit is contained in:
2026-10-22 18:41:56 +03:00
parent 665c37c0b8
commit 3695ab6dfa
32 changed files with 881 additions and 156 deletions
+65
View File
@@ -0,0 +1,65 @@
import base64, json, sys, urllib.error, urllib.request, uuid
from concurrent.futures import ThreadPoolExecutor
BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080"
PWD = "correct-horse-battery-staple"
def call(method, path, body=None, token=None, basic=None, key=None):
data = json.dumps(body).encode() if body is not None else None
req = urllib.request.Request(BASE + path, data=data, method=method)
if body is not None:
req.add_header("Content-Type", "application/json")
if token:
req.add_header("Authorization", "Bearer " + token)
if basic:
req.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode())
if key:
req.add_header("Idempotency-Key", key)
try:
with urllib.request.urlopen(req, timeout=10) as r:
raw = r.read()
return r.status, r.headers.get("X-Instance-Id"), json.loads(raw) if raw else None
except urllib.error.HTTPError as e:
return e.code, e.headers.get("X-Instance-Id"), None
ok = True
def check(name, cond, info=""):
global ok
ok &= bool(cond)
print(("PASS" if cond else "FAIL"), name, info)
name = "lb_" + uuid.uuid4().hex[:10]
s, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "LB check", "password": PWD})
s, _, sess = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}")
token = sess["token"]
# один токен разные реплики
inst = {call("GET", "/api/v1/users/me", token=token)[1] for _ in range(12)}
check("one token served by >=2 replicas", len(inst) >= 2, sorted(inst))
s, _, chat = call("POST", "/api/v1/chats", {"title": "lb", "member_ids": [user["id"]]}, token=token)
path = f"/api/v1/chats/{chat['id']}/messages"
# конкурентные повторы с одним ключом через lb
key = "conc-" + uuid.uuid4().hex
with ThreadPoolExecutor(30) as ex:
res = list(ex.map(lambda _: call("POST", path, {"text": "race"}, token=token, key=key), range(30)))
ids = {r[2]["id"] for r in res if r[2]}
check("concurrent same key: all 201", all(r[0] == 201 for r in res), sorted({r[0] for r in res}))
check("concurrent same key: one message id", len(ids) == 1, len(ids))
check("concurrent requests hit >=2 replicas", len({r[1] for r in res}) >= 2, sorted({r[1] for r in res}))
# последовательный повтор на разных репликах
key2 = "seq-" + uuid.uuid4().hex
seq = [call("POST", path, {"text": "seq"}, token=token, key=key2) for _ in range(6)]
check("sequential repeat: same id", len({r[2]["id"] for r in seq}) == 1)
check("sequential repeat: >=2 replicas", len({r[1] for r in seq}) >= 2, sorted({r[1] for r in seq}))
check("different body, same key -> 409", call("POST", path, {"text": "other"}, token=token, key=key2)[0] == 409)
s, _, page = call("GET", path + "?limit=100", token=token)
check("history has exactly 2 messages", len(page["items"]) == 2, len(page["items"]))
sys.exit(0 if ok else 1)
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
set -eu
mkdir -p evidence
python3 tests/load.py --duration 30 --concurrency 4 \
--output evidence/load-stop-api2.json > evidence/load-stop-api2.txt &
LOAD_PID=$!
sleep 8
date '+%T stop api-2'
docker stop -t 8 mtusi2026lab3-api-2 >/dev/null
docker compose ps > evidence/ps-during-failover.txt
sleep 10
date '+%T start api-2'
docker start mtusi2026lab3-api-2 >/dev/null
wait "$LOAD_PID"
docker compose logs lb --since 3m > evidence/lb-failover.log
cat evidence/load-stop-api2.txt
+91
View File
@@ -0,0 +1,91 @@
import base64, json, subprocess, sys, threading, time, urllib.request, uuid
import os
BASE = os.environ.get("BASE", "http://localhost:8080"); PWD = "correct-horse-battery-staple"
WORKERS, DURATION, STOP_AT, START_AT = 4, 35, 8, 22
def req(method, path, body=None, token=None, basic=None, key=None):
r = urllib.request.Request(BASE + path, method=method,
data=json.dumps(body).encode() if body is not None else None)
if body is not None: r.add_header("Content-Type", "application/json")
if token: r.add_header("Authorization", "Bearer " + token)
if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode())
if key: r.add_header("Idempotency-Key", key)
with urllib.request.urlopen(r, timeout=3) as resp:
return resp.status, resp.headers.get("X-Instance-Id"), json.loads(resp.read() or b"null")
def sh(*a): return subprocess.run(a, capture_output=True, text=True).stdout.split()
arg = sys.argv[1]
names = sorted(sh("docker", "ps", "--filter", "label=com.docker.compose.service=api",
"--format", "{{.Names}}"))
container = names[int(arg) - 1] if arg.isdigit() else arg
stopped_id = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", container],
capture_output=True, text=True).stdout.strip()
assert stopped_id, "container not found"
name = "fo_" + uuid.uuid4().hex[:10]
_, _, u = req("POST", "/api/v1/users", {"username": name, "display_name": "fo", "password": PWD})
_, _, s = req("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}")
tok = s["token"]
_, _, c = req("POST", "/api/v1/chats", {"title": "fo", "member_ids": [u["id"]]}, token=tok)
path = f"/api/v1/chats/{c['id']}/messages"
t0 = time.time()
attempts, confirmed, unconfirmed = [], set(), 0
lock = threading.Lock()
def action():
time.sleep(STOP_AT)
subprocess.run(["docker", "stop", "-t", "8", container], capture_output=True)
time.sleep(max(0, START_AT - (time.time() - t0)))
subprocess.run(["docker", "start", container], capture_output=True)
def worker():
global unconfirmed
while time.time() - t0 < DURATION:
key = uuid.uuid4().hex
done = False
for n in range(3):
ts = time.time() - t0
try:
_, inst, m = req("POST", path, {"text": "x"}, token=tok, key=key)
with lock:
attempts.append((ts, True, inst, n)); confirmed.add(m["id"])
done = True
break
except Exception:
with lock: attempts.append((ts, False, None, n))
time.sleep(0.2)
if not done:
with lock: unconfirmed += 1
threads = [threading.Thread(target=action)] + [threading.Thread(target=worker) for _ in range(WORKERS)]
[t.start() for t in threads]; [t.join() for t in threads]
stored, cur = set(), None
while True:
_, _, page = req("GET", path + "?limit=100" + (f"&cursor={cur}" if cur else ""), token=tok)
stored |= {m["id"] for m in page["items"]}
cur = page["next_cursor"]
if not cur: break
oks = sorted(a[0] for a in attempts if a[1])
gap = max((b - a for a, b in zip(oks, oks[1:])), default=0)
first_fail = [a[0] - STOP_AT for a in attempts if not a[1] and a[3] == 0 and a[0] >= STOP_AT]
pre_stop_errors = sum(1 for a in attempts if not a[1] and a[3] == 0 and a[0] < STOP_AT)
rps = lambda a, b: round(sum(a <= t < b for t in oks) / (b - a), 1)
late = {a[2] for a in attempts if a[1] and STOP_AT + 6 < a[0] < START_AT}
print(json.dumps({
"container": container, "stopped_instance": stopped_id,
"rps_before": rps(0, STOP_AT), "rps_during": rps(STOP_AT, START_AT), "rps_after": rps(START_AT, DURATION),
"first_attempt_errors": len(first_fail),
"error_window_s_after_stop": [round(min(first_fail), 2), round(max(first_fail), 2)] if first_fail else None,
"longest_gap_without_success_s": round(gap, 2),
"stopped_instance_seen_while_down": stopped_id in late,
"confirmed": len(confirmed), "unconfirmed_keys": unconfirmed, "stored": len(stored),
"lost_confirmed": len(confirmed - stored),
"stored_minus_confirmed": len(stored) - len(confirmed),
}, indent=2))
+2 -3
View File
@@ -1,5 +1,4 @@
#!/bin/sh
set -eu
docker compose restart postgres redis api
docker compose restart postgres redis
docker compose restart api lb
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
mkdir -p evidence
docker run --rm --network mtusi2026lab3_edge -v "$PWD":/w python:3.13-alpine \
python /w/tests/load.py --base-url http://lb:8080 --duration 80 --concurrency 4 \
--output /w/evidence/load-rolling.json > evidence/load-rolling.txt &
LOAD=$!
sleep 8
date '+%T rolling start'
./scripts/rolling-update.sh
date '+%T rolling end'
wait "$LOAD"
docker compose ps > evidence/ps-after-rolling.txt
docker compose logs lb --since 4m > evidence/lb-rolling.log
for c in $(docker compose ps -q api); do docker exec "$c" printenv APP_VERSION; done
cat evidence/load-rolling.txt
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
set -eu
N=2
export APP_VERSION="${APP_VERSION:-v$(date +%s)}"
wait_healthy() {
until [ "$(docker compose ps -q api | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$1" ]; do sleep 1; done
}
docker compose build api
for old in $(docker compose ps -q api); do
docker compose up -d --no-deps --no-recreate --scale api=$((N+1)) api
wait_healthy $((N+1))
sleep 5 # LB: rise 2 + обновление DNS
echo "$(date +%T) replacing $(docker inspect -f '{{.Name}}' "$old")"
docker stop -t 20 "$old" >/dev/null
docker rm "$old" >/dev/null
done
wait_healthy $N
echo "$(date +%T) done, APP_VERSION=$APP_VERSION"
+13
View File
@@ -0,0 +1,13 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cid=""
for id in $(docker compose ps -q api); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then cid="$id"; break; fi
done
[ -n "$cid" ] || { echo "container for instance $TARGET_INSTANCE_ID not found" >&2; exit 1; }
docker stop -t 8 "$cid" >/dev/null
echo "stopped $cid" >&2