forked from mtusi2026/mtusi2026lab4
Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
This commit is contained in:
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload → stop the API replica that served the upload → download via another replica.
|
||||
|
||||
Runs one round per replica (the stopped one is restored before the next round).
|
||||
Usage: python scripts/attachment_failover_evidence.py
|
||||
Writes evidence/attachment-failover.json; tokens are never printed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def sh(args) -> str:
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
|
||||
|
||||
|
||||
def request(method, path, body=None, token=None, raw=None, headers=None):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
return resp.code, resp.headers, resp.read()
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"fo_{label}_{uuid.uuid4().hex[:10]}"
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201
|
||||
return json.loads(data)["id"], json.loads(data2)["token"]
|
||||
|
||||
|
||||
def api_containers():
|
||||
rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api",
|
||||
"--filter", "label=com.docker.compose.project=mtusi2026lab4",
|
||||
"--format", "{{.ID}}"]).split()
|
||||
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip():
|
||||
sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows
|
||||
|
||||
|
||||
def observe(token, n=30):
|
||||
return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)}
|
||||
|
||||
|
||||
def wait_two(token, timeout=90):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if len(observe(token, 20)) >= 2:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError("two replicas not observed")
|
||||
|
||||
|
||||
def one_round(index, alice, bob, chat_id, seen_targets):
|
||||
data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
for _ in range(40):
|
||||
status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png",
|
||||
token=alice, raw=data, headers={"Content-Type": "image/png"})
|
||||
assert status == 201, status
|
||||
target = h.get("X-Instance-Id")
|
||||
if target not in seen_targets:
|
||||
break
|
||||
att = json.loads(body)
|
||||
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]},
|
||||
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
assert status == 201
|
||||
before, ids = api_containers()
|
||||
victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target)
|
||||
t0 = time.monotonic()
|
||||
sh(["docker", "stop", "-t", "15", victim])
|
||||
stop_seconds = time.monotonic() - t0
|
||||
downloads, first_ok, errors = [], None, 0
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline and len(downloads) < 20:
|
||||
try:
|
||||
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
|
||||
except OSError:
|
||||
errors += 1
|
||||
continue
|
||||
if status == 200:
|
||||
if first_ok is None:
|
||||
first_ok = time.monotonic() - t0
|
||||
downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest})
|
||||
else:
|
||||
errors += 1
|
||||
after, _ = api_containers()
|
||||
meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0]
|
||||
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
|
||||
linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"])
|
||||
result = {
|
||||
"round": index,
|
||||
"upload_served_by": target,
|
||||
"stopped_container_state_after": after.get(target),
|
||||
"docker_stop_seconds": round(stop_seconds, 2),
|
||||
"first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None,
|
||||
"downloads": len(downloads),
|
||||
"download_instances": sorted({d["instance"] for d in downloads}),
|
||||
"target_served_any_download": any(d["instance"] == target for d in downloads),
|
||||
"all_sha256_match": all(d["sha256_ok"] for d in downloads),
|
||||
"errors_during_window": errors,
|
||||
"metadata_status": meta_status,
|
||||
"message_link_preserved": linked,
|
||||
"containers_before": before,
|
||||
"containers_after": after,
|
||||
}
|
||||
sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"])
|
||||
wait_two(alice)
|
||||
seen_targets.add(target)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
alice_id, alice = actor("a")
|
||||
bob_id, bob = actor("b")
|
||||
status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice)
|
||||
assert status == 201
|
||||
chat_id = json.loads(data)["id"]
|
||||
wait_two(alice)
|
||||
seen = set()
|
||||
rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)]
|
||||
ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"]
|
||||
and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds)
|
||||
report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds}
|
||||
out = ROOT / "evidence" / "attachment-failover.json"
|
||||
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
for r in rounds:
|
||||
print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after",
|
||||
"first_download_ok_after_stop_s", "downloads", "download_instances",
|
||||
"target_served_any_download", "all_sha256_match", "errors_during_window")}))
|
||||
print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Cross-check the student test corpus against independent decoders.
|
||||
|
||||
Corrupted inputs (student-tests/imagegen.py) must NOT decode, valid ones must.
|
||||
PNG → Pillow full load(); JPEG → libjpeg-turbo `djpeg -strict` (what the API runs).
|
||||
Run in a throwaway container (needs Pillow + libjpeg-turbo-progs):
|
||||
|
||||
docker run --rm -v "$PWD:/w" -w /w python:3.12.8-slim-bookworm sh -c \
|
||||
"apt-get update -qq && apt-get install -y -qq libjpeg-turbo-progs >/dev/null; \
|
||||
pip install -q Pillow==11.1.0; python scripts/decoder_crosscheck.py"
|
||||
"""
|
||||
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
# PNG spec errors (PLTE length/placement, unknown critical chunk) that Pillow tolerates;
|
||||
# libpng and the API reject them.
|
||||
SPEC_ONLY = {"plte_not_multiple_of_3", "plte_too_long_for_1bit", "plte_in_grayscale", "unknown_critical_chunk"}
|
||||
|
||||
|
||||
def pillow_ok(data: bytes) -> bool:
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img.load()
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def djpeg_ok(data: bytes) -> bool:
|
||||
with tempfile.NamedTemporaryFile(delete=False) as f:
|
||||
f.write(data)
|
||||
try:
|
||||
cmd = ["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, f.name]
|
||||
return subprocess.run(cmd, capture_output=True).returncode == 0
|
||||
finally:
|
||||
os.unlink(f.name)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
mismatches = 0
|
||||
for name, data in imagegen.corrupted_pngs().items():
|
||||
ok = pillow_ok(data)
|
||||
note = " (PNG spec violation tolerated by Pillow)" if ok and name in SPEC_ONLY else ""
|
||||
mismatches += ok and name not in SPEC_ONLY
|
||||
print(f"corrupt png {name:32} pillow_decodes={ok}{note}")
|
||||
for name, data in imagegen.corrupted_jpegs().items():
|
||||
ok = djpeg_ok(data)
|
||||
mismatches += ok
|
||||
print(f"corrupt jpeg {name:32} djpeg_strict_decodes={ok} pillow_decodes={pillow_ok(data)}")
|
||||
for interlace in (0, 1):
|
||||
for size in ((1, 1), (9, 7), (33, 20)):
|
||||
ok = pillow_ok(imagegen.all_filters_png(*size, interlace=interlace))
|
||||
mismatches += not ok
|
||||
print(f"valid png all_filters interlace={interlace} {size} pillow_decodes={ok}")
|
||||
for name in ("sample.jpg", "progressive.jpg", "gray.jpg"):
|
||||
ok = djpeg_ok(imagegen.fixture(name))
|
||||
mismatches += not ok
|
||||
print(f"valid jpeg {name:32} djpeg_strict_decodes={ok}")
|
||||
print("PASS" if mismatches == 0 else f"FAIL mismatches={mismatches}")
|
||||
return 1 if mismatches else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Whole-stack restart: `docker compose down` (volumes kept) → `up` → same token reads the same file.
|
||||
|
||||
Stronger than scripts/restart.*: every container (API, LB, PostgreSQL, Redis, SeaweedFS,
|
||||
s3-gw, one-shot bootstrap jobs) is removed and recreated; only named volumes survive.
|
||||
Usage: python scripts/full_restart_evidence.py (writes evidence/full-restart.json)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def sh(args) -> str:
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
|
||||
|
||||
|
||||
def request(method, path, body=None, token=None, raw=None, headers=None):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
return resp.code, resp.headers, resp.read()
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"rs_{label}_{uuid.uuid4().hex[:10]}"
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201
|
||||
return json.loads(data)["id"], json.loads(data2)["token"]
|
||||
|
||||
|
||||
def snapshot():
|
||||
rows = sh(COMPOSE + ["ps", "-a", "--format", "{{.Service}} {{.ID}} {{.State}}"]).splitlines()
|
||||
return sorted(rows)
|
||||
|
||||
|
||||
def main():
|
||||
alice_id, alice = actor("a")
|
||||
bob_id, bob = actor("b")
|
||||
status, _, data = request("POST", "/api/v1/chats", {"title": "restart", "member_ids": [alice_id, bob_id]}, token=alice)
|
||||
chat_id = json.loads(data)["id"]
|
||||
payload = imagegen.rgb_png(600, 400, noise=True)
|
||||
status, _, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=keep.png", token=alice,
|
||||
raw=payload, headers={"Content-Type": "image/png"})
|
||||
assert status == 201, status
|
||||
att = json.loads(body)
|
||||
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "переживёт restart\u0000ok", "attachment_ids": [att["id"]]},
|
||||
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
assert status == 201
|
||||
message = json.loads(msg)
|
||||
before = snapshot()
|
||||
volumes = sh(["docker", "volume", "ls", "--filter", "label=com.docker.compose.project=mtusi2026lab4", "--format", "{{.Name}}"]).split()
|
||||
t0 = time.monotonic()
|
||||
sh(COMPOSE + ["down"])
|
||||
down_s = time.monotonic() - t0
|
||||
after_down = snapshot()
|
||||
sh(COMPOSE + ["up", "-d", "--scale", "api=2"])
|
||||
deadline = time.monotonic() + 180
|
||||
while True:
|
||||
try:
|
||||
if request("GET", "/health/ready")[0] == 200 and request("GET", "/api/v1/users/me", token=bob)[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit("FAIL: stack not ready after restart")
|
||||
time.sleep(1)
|
||||
recovery_s = time.monotonic() - t0
|
||||
after = snapshot()
|
||||
meta = json.loads(request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[2])
|
||||
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
|
||||
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
|
||||
s3_init_log = sh(COMPOSE + ["logs", "--no-log-prefix", "s3-init"]).strip().splitlines()[-3:]
|
||||
result = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"volumes_kept": volumes,
|
||||
"containers_before": before,
|
||||
"containers_after_down": after_down,
|
||||
"containers_after_up": after,
|
||||
"down_seconds": round(down_s, 1),
|
||||
"down_to_ready_seconds": round(recovery_s, 1),
|
||||
"same_token_works": True,
|
||||
"metadata_unchanged": meta == att,
|
||||
"download_status": status,
|
||||
"sha256_match": hashlib.sha256(content).hexdigest() == att["sha256"] == hashlib.sha256(payload).hexdigest(),
|
||||
"message_preserved": any(m == message for m in history["items"]),
|
||||
"s3_init_rerun_log_tail": s3_init_log,
|
||||
}
|
||||
ok = result["metadata_unchanged"] and status == 200 and result["sha256_match"] and result["message_preserved"]
|
||||
result["passed"] = ok
|
||||
out = ROOT / "evidence" / "full-restart.json"
|
||||
out.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps({k: result[k] for k in ("down_to_ready_seconds", "metadata_unchanged", "download_status",
|
||||
"sha256_match", "message_preserved", "passed")}))
|
||||
print("WROTE", out)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic secrets in ./secrets (idempotent, prints only file names).
|
||||
|
||||
Existing files are kept, so re-running never rotates credentials of a running
|
||||
stack. secrets/s3_config.json (SeaweedFS identities) is always re-derived from the
|
||||
key files and S3_BUCKET (.env or environment):
|
||||
* admin — Admin/Read/Write/List, used only by the one-shot s3-init;
|
||||
* messenger-api — Read/Write/List on the attachments bucket only, used by API;
|
||||
* no "anonymous" identity → unauthenticated S3 requests are denied.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import string
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
ALNUM = string.ascii_letters + string.digits
|
||||
|
||||
|
||||
def token(length: int, alphabet: str = ALNUM) -> str:
|
||||
return "".join(secrets.choice(alphabet) for _ in range(length))
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
value = os.environ.get("S3_BUCKET", "").strip()
|
||||
env_file = ROOT / ".env"
|
||||
if not value and env_file.is_file():
|
||||
for line in env_file.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("S3_BUCKET="):
|
||||
value = line.split("=", 1)[1].strip()
|
||||
return value or "messenger-attachments"
|
||||
|
||||
|
||||
def ensure(name: str, factory) -> str:
|
||||
path = SECRETS / name
|
||||
if path.is_file() and path.read_text(encoding="utf-8").strip():
|
||||
print("keep secrets/" + name)
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
value = factory()
|
||||
path.write_bytes(value.encode("utf-8"))
|
||||
print("create secrets/" + name)
|
||||
return value
|
||||
|
||||
|
||||
def main() -> int:
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
pg_password = ensure("postgres_password", lambda: token(32))
|
||||
ensure("database_url", lambda: f"postgresql://messenger:{pg_password}@postgres:5432/messenger")
|
||||
ensure("cache_url", lambda: "redis://redis:6379/0")
|
||||
admin_key = ensure("s3_admin_access_key_id", lambda: "ADM" + token(17, string.ascii_uppercase + string.digits))
|
||||
admin_secret = ensure("s3_admin_secret_access_key", lambda: token(40))
|
||||
api_key = ensure("s3_access_key_id", lambda: "API" + token(17, string.ascii_uppercase + string.digits))
|
||||
api_secret = ensure("s3_secret_access_key", lambda: token(40))
|
||||
bucket = bucket_name()
|
||||
config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [{"accessKey": admin_key, "secretKey": admin_secret}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"],
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [{"accessKey": api_key, "secretKey": api_secret}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"],
|
||||
},
|
||||
]
|
||||
}
|
||||
(SECRETS / "s3_config.json").write_bytes(json.dumps(config, indent=1).encode("utf-8"))
|
||||
print(f"write secrets/s3_config.json (bucket {bucket}; identities admin, messenger-api; no anonymous)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Peak RSS of `djpeg -strict -scale 1/8` on worst-case 40 Mpx JPEGs (sizing the API limit).
|
||||
|
||||
Run in a throwaway container with Pillow + libjpeg-turbo-progs:
|
||||
python scripts/jpeg_decode_memory.py gen # writes /tmp/j/*.jpg (8000×5000)
|
||||
python scripts/jpeg_decode_memory.py measure # one fresh child per file
|
||||
The measuring parent stays tiny so RUSAGE_CHILDREN reflects djpeg only.
|
||||
"""
|
||||
|
||||
import os
|
||||
import resource
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
OUT = "/tmp/j"
|
||||
|
||||
|
||||
def gen() -> None:
|
||||
from PIL import Image
|
||||
|
||||
os.makedirs(OUT, exist_ok=True)
|
||||
noise = Image.effect_noise((8000, 5000), 64).convert("L")
|
||||
rgb = Image.merge("RGB", (noise, noise.rotate(180), noise.transpose(Image.FLIP_LEFT_RIGHT)))
|
||||
for name, img, kw in (
|
||||
("baseline_420", rgb, dict(quality=30)),
|
||||
("progressive_420", rgb, dict(quality=30, progressive=True)),
|
||||
("progressive_444", rgb, dict(quality=20, progressive=True, subsampling=0)),
|
||||
("progressive_cmyk", rgb.convert("CMYK"), dict(quality=15, progressive=True, subsampling=0)),
|
||||
):
|
||||
img.save(os.path.join(OUT, name + ".jpg"), "JPEG", **kw)
|
||||
|
||||
|
||||
def measure_one(path: str) -> None:
|
||||
t0 = time.monotonic()
|
||||
proc = subprocess.run(["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path], capture_output=True)
|
||||
took = time.monotonic() - t0
|
||||
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1024
|
||||
print(f"{os.path.basename(path):22} {os.path.getsize(path) / 2**20:6.2f} MiB rc={proc.returncode} "
|
||||
f"{took:5.2f}s djpeg_peak_rss={peak:.1f} MiB", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if sys.argv[1] == "gen":
|
||||
gen()
|
||||
elif sys.argv[1] == "one":
|
||||
measure_one(sys.argv[2])
|
||||
else:
|
||||
for name in sorted(os.listdir(OUT)):
|
||||
subprocess.run([sys.executable, __file__, "one", os.path.join(OUT, name)], check=True)
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LB balancing under mixed traffic: slow uploads interleaved with short GETs.
|
||||
|
||||
Each of 8 uploads (~1 MiB PNG) streams its body over ~4 s; between upload starts the
|
||||
client sends one short GET. Records which replica served every upload and GET.
|
||||
The balance algorithm is read from the running LB config. Results are merged into
|
||||
evidence/lb-interleave.json keyed by algorithm. Tokens are kept in memory only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import collections
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
HOST = os.environ.get("LB_HOST", "127.0.0.1")
|
||||
PORT = int(os.environ.get("LB_PORT", "8080"))
|
||||
UPLOADS = 8
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, headers=None):
|
||||
h = {"Accept": "application/json", **(headers or {})}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
req = urllib.request.Request(f"http://{HOST}:{PORT}{path}", data=data, headers=h, method=method)
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
return resp.headers, json.loads(resp.read() or b"null")
|
||||
|
||||
|
||||
def actor(label):
|
||||
username, password = f"ilv_{label}_{uuid.uuid4().hex[:10]}", "T3st-" + uuid.uuid4().hex
|
||||
_, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
_, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
return user["id"], session["token"]
|
||||
|
||||
|
||||
def balance_algorithm():
|
||||
cfg = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "exec", "-T", "lb",
|
||||
"cat", "/usr/local/etc/haproxy/haproxy.cfg"], cwd=ROOT, text=True)
|
||||
algos = {line.split()[1] for line in cfg.splitlines() if line.strip().startswith("balance ")}
|
||||
return "/".join(sorted(algos))
|
||||
|
||||
|
||||
def main():
|
||||
algo = balance_algorithm()
|
||||
alice, token = actor("a")
|
||||
bob, _ = actor("b")
|
||||
_, chat = call("POST", "/api/v1/chats", {"title": "interleave", "member_ids": [alice, bob]}, token=token)
|
||||
payload = imagegen.rgb_png(700, 500, noise=True)
|
||||
uploads, gets = [], []
|
||||
|
||||
def slow_upload(i):
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
|
||||
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=ilv{i}.png")
|
||||
for k, v in {"Authorization": "Bearer " + token, "Content-Type": "image/png",
|
||||
"Content-Length": str(len(payload))}.items():
|
||||
conn.putheader(k, v)
|
||||
conn.endheaders()
|
||||
step = len(payload) // 20
|
||||
for off in range(0, len(payload), step):
|
||||
conn.send(payload[off:off + step])
|
||||
time.sleep(0.2)
|
||||
resp = conn.getresponse()
|
||||
resp.read()
|
||||
uploads.append({"upload": i, "status": resp.status, "instance": resp.getheader("X-Instance-Id")})
|
||||
|
||||
threads = []
|
||||
for i in range(UPLOADS):
|
||||
th = threading.Thread(target=slow_upload, args=(i,))
|
||||
th.start()
|
||||
threads.append(th)
|
||||
time.sleep(0.15)
|
||||
gets.append(call("GET", "/api/v1/users/me", token=token)[0]["X-Instance-Id"])
|
||||
time.sleep(0.15)
|
||||
for th in threads:
|
||||
th.join()
|
||||
result = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"balance": algo,
|
||||
"uploads": UPLOADS,
|
||||
"upload_bytes": len(payload),
|
||||
"uploads_per_replica": dict(collections.Counter(u["instance"] for u in uploads)),
|
||||
"upload_statuses": dict(collections.Counter(str(u["status"]) for u in uploads)),
|
||||
"short_gets_per_replica": dict(collections.Counter(gets)),
|
||||
}
|
||||
out = ROOT / "evidence" / "lb-interleave.json"
|
||||
report = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {}
|
||||
report[algo] = result
|
||||
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
print("WROTE", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,18 @@
|
||||
@echo off
|
||||
setlocal
|
||||
rem Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); volumes are preserved.
|
||||
cd /d "%~dp0\.."
|
||||
if "%API_REPLICAS%"=="" set API_REPLICAS=2
|
||||
docker compose --env-file .env restart postgres redis s3
|
||||
if errorlevel 1 exit /b 1
|
||||
rem s3-gw shares the s3 network namespace and must be restarted after it.
|
||||
docker compose --env-file .env restart s3-gw
|
||||
if errorlevel 1 exit /b 1
|
||||
rem "up -d" alone leaves running replicas untouched; restart them, then restore the scale.
|
||||
docker compose --env-file .env restart api
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --no-deps --scale api=%API_REPLICAS% api
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env restart lb
|
||||
if errorlevel 1 exit /b 1
|
||||
exit /b 0
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); preserve volumes.
|
||||
cd "$(dirname "$0")/.."
|
||||
REPLICAS="${API_REPLICAS:-2}"
|
||||
docker compose --env-file .env restart postgres redis s3
|
||||
# s3-gw shares the s3 network namespace and must be restarted after it.
|
||||
docker compose --env-file .env restart s3-gw
|
||||
# "up -d" alone leaves running replicas untouched; restart them, then restore the scale
|
||||
# (migrate/s3-init already applied; skip one-shots).
|
||||
docker compose --env-file .env restart api
|
||||
docker compose --env-file .env up -d --no-deps --scale "api=${REPLICAS}" api
|
||||
docker compose --env-file .env restart lb
|
||||
exit 0
|
||||
@@ -0,0 +1,51 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions EnableDelayedExpansion
|
||||
rem Grade-4 rolling replacement: replace API containers one-by-one under LB.
|
||||
rem Usage: scripts\rolling-update.cmd
|
||||
cd /d "%~dp0\.."
|
||||
set "REPLICAS=%API_REPLICAS%"
|
||||
if "%REPLICAS%"=="" set "REPLICAS=2"
|
||||
set "URL=http://127.0.0.1:8080/health/ready"
|
||||
|
||||
set "IDS="
|
||||
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do set "IDS=!IDS! %%I"
|
||||
set "KNOWN=%IDS%"
|
||||
set /a COUNT=0
|
||||
for %%I in (%IDS%) do (
|
||||
set /a COUNT+=1
|
||||
echo rolling: replacing %%I
|
||||
rem -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
|
||||
docker stop -t 15 %%I >nul || exit /b 1
|
||||
docker rm %%I >nul || exit /b 1
|
||||
rem --no-recreate: the surviving replica keeps serving even if the image/config changed.
|
||||
docker compose --env-file .env up -d --no-deps --no-recreate --scale api=%REPLICAS% api || exit /b 1
|
||||
call :wait_new || exit /b 1
|
||||
)
|
||||
echo rolling update finished (%COUNT% steps, scale=%REPLICAS%)
|
||||
exit /b 0
|
||||
|
||||
:wait_new
|
||||
set "NEW="
|
||||
for /f "usebackq delims=" %%J in (`docker compose --env-file .env ps -q api`) do (
|
||||
if "!KNOWN:%%J=!"=="!KNOWN!" set "NEW=%%J"
|
||||
)
|
||||
if "%NEW%"=="" (
|
||||
echo rolling: expected one new replica >&2
|
||||
exit /b 1
|
||||
)
|
||||
set "KNOWN=%KNOWN% %NEW%"
|
||||
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %NEW%`) do set "HOST=%%H"
|
||||
rem Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
|
||||
set /a TRIES=0
|
||||
:poll
|
||||
curl.exe -s -D - -o NUL "%URL%" | findstr /I /B /C:"x-instance-id: %HOST%" >nul && goto ready
|
||||
set /a TRIES+=1
|
||||
if %TRIES% geq 90 (
|
||||
echo rolling: %HOST% not in LB rotation >&2
|
||||
exit /b 1
|
||||
)
|
||||
ping -n 2 127.0.0.1 >nul
|
||||
goto poll
|
||||
:ready
|
||||
echo rolling: step %COUNT% ready (%HOST%)
|
||||
exit /b 0
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Grade-4 rolling replacement: replace API containers one-by-one under LB.
|
||||
# Usage: scripts/rolling-update.sh
|
||||
cd "$(dirname "$0")/.."
|
||||
REPLICAS="${API_REPLICAS:-2}"
|
||||
URL="http://127.0.0.1:8080/health/ready"
|
||||
|
||||
ids=$(docker compose --env-file .env ps -q api | tr -d '\r')
|
||||
known="$ids"
|
||||
count=0
|
||||
for id in $ids; do
|
||||
[ -n "$id" ] || continue
|
||||
count=$((count + 1))
|
||||
printf '%s\n' "rolling: replacing $id"
|
||||
# -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
|
||||
docker stop -t 15 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
|
||||
docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=${REPLICAS}" api
|
||||
new=$(docker compose --env-file .env ps -q api | tr -d '\r' | grep -v -F "$known" || true)
|
||||
[ "$(printf '%s\n' "$new" | grep -c .)" -eq 1 ] || { printf '%s\n' "rolling: expected one new replica" >&2; exit 1; }
|
||||
known="$known
|
||||
$new"
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$new" | tr -d '\r')
|
||||
# Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
|
||||
i=0
|
||||
until curl -s -D - -o /dev/null "$URL" | tr -d '\r' | grep -qi "^x-instance-id: ${host}\$"; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -lt 180 ] || { printf '%s\n' "rolling: $host not in LB rotation" >&2; exit 1; }
|
||||
sleep 0.5
|
||||
done
|
||||
printf '%s\n' "rolling: step $count ready ($host)"
|
||||
done
|
||||
printf '%s\n' "rolling update finished ($count steps, scale=${REPLICAS})"
|
||||
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4: closed-loop POSTs during rolling API replacement."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
REPLICAS = os.environ.get("API_REPLICAS", "2")
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, headers=None):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
h["Content-Type"] = "application/json"
|
||||
if headers:
|
||||
h.update(headers)
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=5)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
except Exception as exc:
|
||||
return {"status": 0, "error": type(exc).__name__}
|
||||
raw = resp.read()
|
||||
js = None
|
||||
if raw and resp.headers.get_content_type() == "application/json":
|
||||
js = json.loads(raw.decode("utf-8"))
|
||||
return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")}
|
||||
|
||||
|
||||
def api_ids():
|
||||
out = subprocess.check_output(
|
||||
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
)
|
||||
return out.split()
|
||||
|
||||
|
||||
def hostname(cid):
|
||||
return subprocess.check_output(
|
||||
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
|
||||
).strip()
|
||||
|
||||
|
||||
def replace_replica(old_id, known):
|
||||
"""Stop+remove one replica, create a fresh one, wait until the LB routes to it."""
|
||||
subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL)
|
||||
subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL)
|
||||
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
|
||||
subprocess.check_call(
|
||||
["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate",
|
||||
"--scale", f"api={REPLICAS}", "api"],
|
||||
cwd=ROOT,
|
||||
)
|
||||
new = [cid for cid in api_ids() if cid not in known]
|
||||
if len(new) != 1:
|
||||
raise RuntimeError(f"expected one new replica, got {new}")
|
||||
host = hostname(new[0])
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
r = call("GET", "/health/ready")
|
||||
if r.get("status") == 200 and r.get("instance") == host:
|
||||
return new[0], host
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"new replica {host} not in LB rotation after 90s")
|
||||
|
||||
|
||||
def bootstrap():
|
||||
uname = "roll_" + uuid.uuid4().hex[:10]
|
||||
pw = "T3st-" + uuid.uuid4().hex
|
||||
user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw})
|
||||
assert user["status"] == 201, user
|
||||
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
|
||||
tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"]
|
||||
peer_u = "rollb_" + uuid.uuid4().hex[:10]
|
||||
peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"]
|
||||
chat = call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]},
|
||||
token=tok,
|
||||
)["json"]
|
||||
return tok, chat["id"]
|
||||
|
||||
|
||||
def main():
|
||||
tok, chat_id = bootstrap()
|
||||
stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []}
|
||||
stop = threading.Event()
|
||||
|
||||
def worker():
|
||||
while not stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
t0 = time.monotonic()
|
||||
r = call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat_id}/messages",
|
||||
{"text": "roll-" + key},
|
||||
token=tok,
|
||||
headers={"Idempotency-Key": key},
|
||||
)
|
||||
dt = (time.monotonic() - t0) * 1000
|
||||
if r.get("status") == 201:
|
||||
stats["ok"] += 1
|
||||
stats["latencies_ms"].append(dt)
|
||||
else:
|
||||
stats["err"] += 1
|
||||
msg = ((r.get("json") or {}).get("error") or {}).get("message", "")
|
||||
code = f"{r.get('status') or r.get('error')} {msg}".strip()
|
||||
stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1
|
||||
time.sleep(0.05)
|
||||
|
||||
ids = api_ids()
|
||||
known = set(ids)
|
||||
replaced = []
|
||||
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
|
||||
started = time.monotonic()
|
||||
for th in threads:
|
||||
th.start()
|
||||
time.sleep(2) # warm-up
|
||||
# Rolling: replace each original replica in turn; the next one is stopped
|
||||
# only after the LB already routes requests to the fresh container.
|
||||
for cid in ids:
|
||||
old_host = hostname(cid)
|
||||
new_id, new_host = replace_replica(cid, known)
|
||||
known.add(new_id)
|
||||
replaced.append({"old": old_host, "new": new_host})
|
||||
time.sleep(2)
|
||||
time.sleep(2)
|
||||
stop.set()
|
||||
for th in threads:
|
||||
th.join(timeout=5)
|
||||
duration = time.monotonic() - started
|
||||
lats = sorted(stats["latencies_ms"])
|
||||
def pct(p):
|
||||
if not lats:
|
||||
return None
|
||||
return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2)
|
||||
summary = {
|
||||
"successful": stats["ok"],
|
||||
"errors": stats["err"],
|
||||
"errors_by_status": stats["err_by_status"],
|
||||
"p50_ms": pct(0.50),
|
||||
"p95_ms": pct(0.95),
|
||||
"p99_ms": pct(0.99),
|
||||
"duration_s": round(duration, 1),
|
||||
"replicas_rolled": len(replaced),
|
||||
"replaced": replaced,
|
||||
}
|
||||
out = ROOT / "evidence" / "rolling-load.json"
|
||||
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
print("WROTE", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade-4 evidence: stop each observed API replica in turn; restore between steps."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, headers=None, expected=None):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
h["Content-Type"] = "application/json"
|
||||
if headers:
|
||||
h.update(headers)
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=5)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
raw = resp.read()
|
||||
status = resp.code
|
||||
if expected is not None and status != expected:
|
||||
raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}")
|
||||
js = None
|
||||
if raw and resp.headers.get_content_type() == "application/json":
|
||||
js = json.loads(raw.decode("utf-8"))
|
||||
return {"status": status, "headers": resp.headers, "json": js}
|
||||
|
||||
|
||||
def header(resp, name: str) -> str:
|
||||
return resp["headers"].get(name) or resp["headers"].get(name.lower()) or ""
|
||||
|
||||
|
||||
def restore_api(target: str):
|
||||
replicas = os.environ.get("API_REPLICAS", "2")
|
||||
subprocess.check_call(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"--env-file",
|
||||
".env",
|
||||
"up",
|
||||
"-d",
|
||||
"--no-deps",
|
||||
"--scale",
|
||||
f"api={replicas}",
|
||||
"--remove-orphans",
|
||||
"api",
|
||||
],
|
||||
cwd=ROOT,
|
||||
)
|
||||
# Public ready is 200 via the surviving replica; wait until the LB routes to the restored one.
|
||||
for _ in range(180):
|
||||
try:
|
||||
if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target:
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"replica {target} not back in LB rotation after restore")
|
||||
|
||||
|
||||
def one_round(round_no: int) -> dict:
|
||||
uname = "seq_" + uuid.uuid4().hex[:10]
|
||||
pw = "T3st-" + uuid.uuid4().hex
|
||||
user = call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": uname, "display_name": "Seq", "password": pw},
|
||||
expected=201,
|
||||
)["json"]
|
||||
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
|
||||
tok = call(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": "Basic " + cred},
|
||||
expected=201,
|
||||
)["json"]["token"]
|
||||
peer_u = "seqb_" + uuid.uuid4().hex[:10]
|
||||
peer = call(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
{"username": peer_u, "display_name": "SeqB", "password": pw},
|
||||
expected=201,
|
||||
)["json"]
|
||||
chat = call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
{"title": "seq", "member_ids": [user["id"], peer["id"]]},
|
||||
token=tok,
|
||||
expected=201,
|
||||
)["json"]
|
||||
|
||||
observed = set()
|
||||
for _ in range(40):
|
||||
observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id"))
|
||||
if len(observed) < 2:
|
||||
raise RuntimeError(f"need ≥2 instances, got {observed}")
|
||||
target = sorted(observed)[round_no % len(observed)]
|
||||
|
||||
env = dict(os.environ)
|
||||
env["TARGET_INSTANCE_ID"] = target
|
||||
script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh")
|
||||
started = time.monotonic()
|
||||
proc = subprocess.Popen([str(script)], env=env, cwd=ROOT)
|
||||
ok = fail = 0
|
||||
first = None
|
||||
after = []
|
||||
while time.monotonic() - started < 20:
|
||||
code = proc.poll()
|
||||
try:
|
||||
key = uuid.uuid4().hex
|
||||
r = call(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
{"text": "seq-" + key},
|
||||
token=tok,
|
||||
headers={"Idempotency-Key": key},
|
||||
expected=201,
|
||||
)
|
||||
ok += 1
|
||||
if code == 0:
|
||||
if first is None:
|
||||
first = time.monotonic() - started
|
||||
after.append(header(r, "X-Instance-Id"))
|
||||
except Exception:
|
||||
fail += 1
|
||||
time.sleep(0.1)
|
||||
if proc.poll() is None:
|
||||
proc.wait(timeout=10)
|
||||
result = {
|
||||
"round": round_no,
|
||||
"observed_before": sorted(observed),
|
||||
"target": target,
|
||||
"successful_writes": ok,
|
||||
"failed_attempts": fail,
|
||||
"first_recovery_seconds": round(first, 3) if first is not None else None,
|
||||
"target_absent_in_last5": target not in after[-5:],
|
||||
}
|
||||
t0 = time.monotonic()
|
||||
restore_api(target)
|
||||
result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
results = []
|
||||
for i in range(2):
|
||||
results.append(one_round(i))
|
||||
print(json.dumps(results[-1], ensure_ascii=False))
|
||||
out = ROOT / "evidence" / "seq-failover.json"
|
||||
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print("WROTE", out)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,17 @@
|
||||
@echo off
|
||||
setlocal EnableExtensions
|
||||
cd /d "%~dp0\.."
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo stop-one: TARGET_INSTANCE_ID required >&2
|
||||
exit /b 1
|
||||
)
|
||||
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do (
|
||||
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %%I`) do (
|
||||
if /I "%%H"=="%TARGET_INSTANCE_ID%" (
|
||||
docker stop -t 15 %%I >nul
|
||||
exit /b 0
|
||||
)
|
||||
)
|
||||
)
|
||||
echo stop-one: replica not found >&2
|
||||
exit /b 1
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Stop ONE API replica matching TARGET_INSTANCE_ID (= X-Instance-Id / hostname).
|
||||
# Leave it stopped; do not touch DB/Redis/LB/volumes.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
found=""
|
||||
# Compose project containers for service api
|
||||
ids=$(docker compose --env-file .env ps -q api 2>/dev/null || true)
|
||||
for id in $ids; do
|
||||
[ -n "$id" ] || continue
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$id
|
||||
break
|
||||
fi
|
||||
iid=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$id" | sed -n 's/^INSTANCE_ID=//p' | head -n1)
|
||||
if [ -n "$iid" ] && [ "$iid" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$id
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
# Fallback: any running container with matching hostname (scaled names differ).
|
||||
for id in $(docker ps -q --filter "ancestor=mtusi-messenger:lab4"); do
|
||||
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
|
||||
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
|
||||
found=$id
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -z "$found" ]; then
|
||||
printf '%s\n' "stop-one: replica not found for TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Graceful stop (= stop_grace_period): on SIGTERM the API reports not-ready but keeps
|
||||
# serving for a 3 s drain window, then finishes in-flight requests and exits (~3.5 s).
|
||||
docker stop -t 15 "$found" >/dev/null
|
||||
exit 0
|
||||
@@ -0,0 +1,12 @@
|
||||
@echo off
|
||||
setlocal
|
||||
cd /d "%~dp0\.."
|
||||
if "%API_REPLICAS%"=="" set API_REPLICAS=2
|
||||
python scripts\init_secrets.py
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env config --quiet
|
||||
if errorlevel 1 exit /b 1
|
||||
docker compose --env-file .env up -d --build --scale api=%API_REPLICAS%
|
||||
if errorlevel 1 exit /b 1
|
||||
echo stack up: api scale=%API_REPLICAS%, entry http://127.0.0.1:8080
|
||||
exit /b 0
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Bootstrap lab4 stack with ≥2 API replicas behind LB, PostgreSQL, Redis and private S3.
|
||||
cd "$(dirname "$0")/.."
|
||||
REPLICAS="${API_REPLICAS:-2}"
|
||||
python3 scripts/init_secrets.py
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --scale "api=${REPLICAS}"
|
||||
printf '%s\n' "stack up: api scale=${REPLICAS}, entry http://127.0.0.1:8080"
|
||||
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
|
||||
|
||||
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
|
||||
file) keeps the API process RSS growth far below N × file size; total in-flight
|
||||
work per replica is capped by MAX_CONCURRENT_UPLOADS.
|
||||
|
||||
Method (per concurrency level, fresh replicas):
|
||||
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
|
||||
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
|
||||
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
|
||||
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
|
||||
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
|
||||
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
|
||||
|
||||
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
|
||||
Writes evidence/upload-memory.json. Tokens are kept in memory only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def sh(args, check=True) -> str:
|
||||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
|
||||
if check and proc.returncode != 0:
|
||||
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
|
||||
h = {"Accept": "application/json"}
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=timeout)
|
||||
except urllib.error.HTTPError as err:
|
||||
resp = err
|
||||
data = resp.read()
|
||||
return resp.code, resp.headers, data
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201, status
|
||||
user = json.loads(data)
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201, status
|
||||
return user["id"], json.loads(data)["token"]
|
||||
|
||||
|
||||
def wait_ready(timeout=120):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
if request("GET", "/health/ready", timeout=3)[0] == 200:
|
||||
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
|
||||
if len(seen) >= 2:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError("stack not ready")
|
||||
|
||||
|
||||
def containers():
|
||||
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
|
||||
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
|
||||
|
||||
|
||||
def proc_status(cid):
|
||||
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
|
||||
fields = {}
|
||||
for line in out.splitlines():
|
||||
if line.startswith(("VmRSS", "VmHWM")):
|
||||
k, v = line.split(":", 1)
|
||||
fields[k] = int(v.split()[0]) // 1024 # MiB
|
||||
return fields
|
||||
|
||||
|
||||
def cgroup(cid):
|
||||
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
|
||||
vals = dict(line.split() for line in stat.splitlines() if line)
|
||||
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
|
||||
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
|
||||
mib = 1024 * 1024
|
||||
return {
|
||||
"anon_mib": round(int(vals["anon"]) / mib, 1),
|
||||
"file_mib": round(int(vals["file"]) / mib, 1),
|
||||
"current_mib": round(int(current) / mib, 1),
|
||||
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
|
||||
}
|
||||
|
||||
|
||||
def sampler(cid, stop, out):
|
||||
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
|
||||
while not stop.is_set():
|
||||
try:
|
||||
c = cgroup(cid)
|
||||
for k in ("anon_mib", "file_mib", "current_mib"):
|
||||
best[k] = max(best[k], c[k])
|
||||
best["samples"] += 1
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(0.25)
|
||||
out[cid] = best
|
||||
|
||||
|
||||
def pct(values, q):
|
||||
values = sorted(values)
|
||||
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
|
||||
|
||||
|
||||
def run_level(n, payload, chat_id, token):
|
||||
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
|
||||
# s3-gw shares the s3 network namespace and must be restarted after it.
|
||||
sh(COMPOSE + ["restart", "s3"])
|
||||
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
|
||||
wait_ready()
|
||||
by_host = containers()
|
||||
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
|
||||
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
|
||||
# Warm-up: at least one upload served by every replica.
|
||||
served = set()
|
||||
for _ in range(20):
|
||||
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
|
||||
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
|
||||
served.add(h.get("X-Instance-Id"))
|
||||
if served >= set(by_host):
|
||||
break
|
||||
baseline = {}
|
||||
for host, cid in by_host.items():
|
||||
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
|
||||
baseline[host] = {**proc_status(cid), **cgroup(cid)}
|
||||
lb_baseline = cgroup(lb)
|
||||
s3_baseline = cgroup(s3c)
|
||||
stop = threading.Event()
|
||||
sampled = {}
|
||||
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
|
||||
for t in threads:
|
||||
t.start()
|
||||
time.sleep(0.5)
|
||||
barrier = threading.Barrier(n)
|
||||
results = [None] * n
|
||||
|
||||
def one(i):
|
||||
barrier.wait()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
|
||||
token=token, raw=payload, headers={"Content-Type": "image/png"})
|
||||
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
|
||||
except OSError as exc:
|
||||
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
|
||||
|
||||
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
|
||||
started = time.monotonic()
|
||||
for w in workers:
|
||||
w.start()
|
||||
for w in workers:
|
||||
w.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(1.0)
|
||||
stop.set()
|
||||
for t in threads:
|
||||
t.join()
|
||||
replicas = {}
|
||||
for host, cid in by_host.items():
|
||||
after = proc_status(cid)
|
||||
cg = cgroup(cid)
|
||||
replicas[host] = {
|
||||
"uploads_served": sum(1 for r in results if r[2] == host),
|
||||
"rss_baseline_mib": baseline[host]["VmRSS"],
|
||||
"rss_peak_vmhwm_mib": after["VmHWM"],
|
||||
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
|
||||
"cgroup_peak_mib": cg["peak_mib"],
|
||||
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
|
||||
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
|
||||
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
|
||||
"samples": sampled.get(cid, {}).get("samples"),
|
||||
}
|
||||
latencies = [r[1] for r in results if r[0] == 201]
|
||||
statuses = {}
|
||||
for r in results:
|
||||
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
|
||||
per_replica_max = max(v["uploads_served"] for v in replicas.values())
|
||||
return {
|
||||
"concurrency": n,
|
||||
"payload_bytes": len(payload),
|
||||
"statuses": statuses,
|
||||
"wall_seconds": round(wall, 3),
|
||||
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
|
||||
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
|
||||
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
|
||||
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
|
||||
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
|
||||
"replicas": replicas,
|
||||
"lb": {
|
||||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
|
||||
"baseline_current_mib": lb_baseline["current_mib"],
|
||||
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
|
||||
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
|
||||
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
|
||||
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
|
||||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
|
||||
},
|
||||
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
|
||||
}
|
||||
|
||||
|
||||
def _s3_report(cid, baseline, sampled):
|
||||
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
|
||||
return {
|
||||
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
|
||||
"baseline_current_mib": baseline["current_mib"],
|
||||
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
|
||||
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
|
||||
"sampled_max_anon_mib": sampled.get("anon_mib"),
|
||||
"sampled_max_current_mib": sampled.get("current_mib"),
|
||||
"running": running,
|
||||
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
|
||||
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
|
||||
p.add_argument("--size-mib", type=float, default=9.5)
|
||||
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
|
||||
args = p.parse_args()
|
||||
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
|
||||
_, token = actor("a")
|
||||
bob_id, _ = actor("b")
|
||||
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
|
||||
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
|
||||
assert status == 201, status
|
||||
chat_id = json.loads(data)["id"]
|
||||
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
|
||||
list(containers().values())[0]]).split()
|
||||
report = {
|
||||
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
|
||||
"api_cpus": int(limits[1]) / 1e9,
|
||||
"max_concurrent_uploads_per_replica": 8,
|
||||
"levels": [],
|
||||
}
|
||||
for n in args.levels:
|
||||
level = run_level(n, payload, chat_id, token)
|
||||
report["levels"].append(level)
|
||||
print(json.dumps(level, ensure_ascii=False), flush=True)
|
||||
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print("WROTE", args.output)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user