Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.

Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
This commit is contained in:
Mikhail Verkovykh
2026-11-06 19:48:37 +03:00
parent ec540a1a72
commit 728e53cc55
94 changed files with 8821 additions and 43 deletions
+168
View File
@@ -0,0 +1,168 @@
#!/usr/bin/env python3
"""Upload → stop the API replica that served the upload → download via another replica.
Runs one round per replica (the stopped one is restored before the next round).
Usage: python scripts/attachment_failover_evidence.py
Writes evidence/attachment-failover.json; tokens are never printed.
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
def request(method, path, body=None, token=None, raw=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=10)
except urllib.error.HTTPError as err:
resp = err
return resp.code, resp.headers, resp.read()
def actor(label):
username = f"fo_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201
return json.loads(data)["id"], json.loads(data2)["token"]
def api_containers():
rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.project=mtusi2026lab4",
"--format", "{{.ID}}"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip():
sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows
def observe(token, n=30):
return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)}
def wait_two(token, timeout=90):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if len(observe(token, 20)) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("two replicas not observed")
def one_round(index, alice, bob, chat_id, seen_targets):
data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible
digest = hashlib.sha256(data).hexdigest()
for _ in range(40):
status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png",
token=alice, raw=data, headers={"Content-Type": "image/png"})
assert status == 201, status
target = h.get("X-Instance-Id")
if target not in seen_targets:
break
att = json.loads(body)
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]},
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
assert status == 201
before, ids = api_containers()
victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target)
t0 = time.monotonic()
sh(["docker", "stop", "-t", "15", victim])
stop_seconds = time.monotonic() - t0
downloads, first_ok, errors = [], None, 0
deadline = time.monotonic() + 30
while time.monotonic() < deadline and len(downloads) < 20:
try:
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
except OSError:
errors += 1
continue
if status == 200:
if first_ok is None:
first_ok = time.monotonic() - t0
downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest})
else:
errors += 1
after, _ = api_containers()
meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0]
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"])
result = {
"round": index,
"upload_served_by": target,
"stopped_container_state_after": after.get(target),
"docker_stop_seconds": round(stop_seconds, 2),
"first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None,
"downloads": len(downloads),
"download_instances": sorted({d["instance"] for d in downloads}),
"target_served_any_download": any(d["instance"] == target for d in downloads),
"all_sha256_match": all(d["sha256_ok"] for d in downloads),
"errors_during_window": errors,
"metadata_status": meta_status,
"message_link_preserved": linked,
"containers_before": before,
"containers_after": after,
}
sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"])
wait_two(alice)
seen_targets.add(target)
return result
def main():
alice_id, alice = actor("a")
bob_id, bob = actor("b")
status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice)
assert status == 201
chat_id = json.loads(data)["id"]
wait_two(alice)
seen = set()
rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)]
ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"]
and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds)
report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds}
out = ROOT / "evidence" / "attachment-failover.json"
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
for r in rounds:
print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after",
"first_download_ok_after_stop_s", "downloads", "download_instances",
"target_served_any_download", "all_sha256_match", "errors_during_window")}))
print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out)
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
+72
View File
@@ -0,0 +1,72 @@
#!/usr/bin/env python3
"""Cross-check the student test corpus against independent decoders.
Corrupted inputs (student-tests/imagegen.py) must NOT decode, valid ones must.
PNG → Pillow full load(); JPEG → libjpeg-turbo `djpeg -strict` (what the API runs).
Run in a throwaway container (needs Pillow + libjpeg-turbo-progs):
docker run --rm -v "$PWD:/w" -w /w python:3.12.8-slim-bookworm sh -c \
"apt-get update -qq && apt-get install -y -qq libjpeg-turbo-progs >/dev/null; \
pip install -q Pillow==11.1.0; python scripts/decoder_crosscheck.py"
"""
import io
import os
import subprocess
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "student-tests"))
import imagegen # noqa: E402
from PIL import Image # noqa: E402
# PNG spec errors (PLTE length/placement, unknown critical chunk) that Pillow tolerates;
# libpng and the API reject them.
SPEC_ONLY = {"plte_not_multiple_of_3", "plte_too_long_for_1bit", "plte_in_grayscale", "unknown_critical_chunk"}
def pillow_ok(data: bytes) -> bool:
try:
img = Image.open(io.BytesIO(data))
img.load()
return True
except Exception:
return False
def djpeg_ok(data: bytes) -> bool:
with tempfile.NamedTemporaryFile(delete=False) as f:
f.write(data)
try:
cmd = ["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, f.name]
return subprocess.run(cmd, capture_output=True).returncode == 0
finally:
os.unlink(f.name)
def main() -> int:
mismatches = 0
for name, data in imagegen.corrupted_pngs().items():
ok = pillow_ok(data)
note = " (PNG spec violation tolerated by Pillow)" if ok and name in SPEC_ONLY else ""
mismatches += ok and name not in SPEC_ONLY
print(f"corrupt png {name:32} pillow_decodes={ok}{note}")
for name, data in imagegen.corrupted_jpegs().items():
ok = djpeg_ok(data)
mismatches += ok
print(f"corrupt jpeg {name:32} djpeg_strict_decodes={ok} pillow_decodes={pillow_ok(data)}")
for interlace in (0, 1):
for size in ((1, 1), (9, 7), (33, 20)):
ok = pillow_ok(imagegen.all_filters_png(*size, interlace=interlace))
mismatches += not ok
print(f"valid png all_filters interlace={interlace} {size} pillow_decodes={ok}")
for name in ("sample.jpg", "progressive.jpg", "gray.jpg"):
ok = djpeg_ok(imagegen.fixture(name))
mismatches += not ok
print(f"valid jpeg {name:32} djpeg_strict_decodes={ok}")
print("PASS" if mismatches == 0 else f"FAIL mismatches={mismatches}")
return 1 if mismatches else 0
if __name__ == "__main__":
sys.exit(main())
+130
View File
@@ -0,0 +1,130 @@
#!/usr/bin/env python3
"""Whole-stack restart: `docker compose down` (volumes kept) → `up` → same token reads the same file.
Stronger than scripts/restart.*: every container (API, LB, PostgreSQL, Redis, SeaweedFS,
s3-gw, one-shot bootstrap jobs) is removed and recreated; only named volumes survive.
Usage: python scripts/full_restart_evidence.py (writes evidence/full-restart.json)
"""
from __future__ import annotations
import base64
import hashlib
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args) -> str:
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout
def request(method, path, body=None, token=None, raw=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=10)
except urllib.error.HTTPError as err:
resp = err
return resp.code, resp.headers, resp.read()
def actor(label):
username = f"rs_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201
return json.loads(data)["id"], json.loads(data2)["token"]
def snapshot():
rows = sh(COMPOSE + ["ps", "-a", "--format", "{{.Service}} {{.ID}} {{.State}}"]).splitlines()
return sorted(rows)
def main():
alice_id, alice = actor("a")
bob_id, bob = actor("b")
status, _, data = request("POST", "/api/v1/chats", {"title": "restart", "member_ids": [alice_id, bob_id]}, token=alice)
chat_id = json.loads(data)["id"]
payload = imagegen.rgb_png(600, 400, noise=True)
status, _, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=keep.png", token=alice,
raw=payload, headers={"Content-Type": "image/png"})
assert status == 201, status
att = json.loads(body)
status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "переживёт restart\u0000ok", "attachment_ids": [att["id"]]},
token=alice, headers={"Idempotency-Key": uuid.uuid4().hex})
assert status == 201
message = json.loads(msg)
before = snapshot()
volumes = sh(["docker", "volume", "ls", "--filter", "label=com.docker.compose.project=mtusi2026lab4", "--format", "{{.Name}}"]).split()
t0 = time.monotonic()
sh(COMPOSE + ["down"])
down_s = time.monotonic() - t0
after_down = snapshot()
sh(COMPOSE + ["up", "-d", "--scale", "api=2"])
deadline = time.monotonic() + 180
while True:
try:
if request("GET", "/health/ready")[0] == 200 and request("GET", "/api/v1/users/me", token=bob)[0] == 200:
break
except OSError:
pass
if time.monotonic() > deadline:
raise SystemExit("FAIL: stack not ready after restart")
time.sleep(1)
recovery_s = time.monotonic() - t0
after = snapshot()
meta = json.loads(request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[2])
status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob)
history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2])
s3_init_log = sh(COMPOSE + ["logs", "--no-log-prefix", "s3-init"]).strip().splitlines()[-3:]
result = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"volumes_kept": volumes,
"containers_before": before,
"containers_after_down": after_down,
"containers_after_up": after,
"down_seconds": round(down_s, 1),
"down_to_ready_seconds": round(recovery_s, 1),
"same_token_works": True,
"metadata_unchanged": meta == att,
"download_status": status,
"sha256_match": hashlib.sha256(content).hexdigest() == att["sha256"] == hashlib.sha256(payload).hexdigest(),
"message_preserved": any(m == message for m in history["items"]),
"s3_init_rerun_log_tail": s3_init_log,
}
ok = result["metadata_unchanged"] and status == 200 and result["sha256_match"] and result["message_preserved"]
result["passed"] = ok
out = ROOT / "evidence" / "full-restart.json"
out.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps({k: result[k] for k in ("down_to_ready_seconds", "metadata_unchanged", "download_status",
"sha256_match", "message_preserved", "passed")}))
print("WROTE", out)
return 0 if ok else 1
if __name__ == "__main__":
sys.exit(main())
+80
View File
@@ -0,0 +1,80 @@
#!/usr/bin/env python3
"""Create local synthetic secrets in ./secrets (idempotent, prints only file names).
Existing files are kept, so re-running never rotates credentials of a running
stack. secrets/s3_config.json (SeaweedFS identities) is always re-derived from the
key files and S3_BUCKET (.env or environment):
* admin — Admin/Read/Write/List, used only by the one-shot s3-init;
* messenger-api — Read/Write/List on the attachments bucket only, used by API;
* no "anonymous" identity → unauthenticated S3 requests are denied.
"""
from __future__ import annotations
import json
import os
import secrets
import string
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
ALNUM = string.ascii_letters + string.digits
def token(length: int, alphabet: str = ALNUM) -> str:
return "".join(secrets.choice(alphabet) for _ in range(length))
def bucket_name() -> str:
value = os.environ.get("S3_BUCKET", "").strip()
env_file = ROOT / ".env"
if not value and env_file.is_file():
for line in env_file.read_text(encoding="utf-8").splitlines():
if line.startswith("S3_BUCKET="):
value = line.split("=", 1)[1].strip()
return value or "messenger-attachments"
def ensure(name: str, factory) -> str:
path = SECRETS / name
if path.is_file() and path.read_text(encoding="utf-8").strip():
print("keep secrets/" + name)
return path.read_text(encoding="utf-8").strip()
value = factory()
path.write_bytes(value.encode("utf-8"))
print("create secrets/" + name)
return value
def main() -> int:
SECRETS.mkdir(exist_ok=True)
pg_password = ensure("postgres_password", lambda: token(32))
ensure("database_url", lambda: f"postgresql://messenger:{pg_password}@postgres:5432/messenger")
ensure("cache_url", lambda: "redis://redis:6379/0")
admin_key = ensure("s3_admin_access_key_id", lambda: "ADM" + token(17, string.ascii_uppercase + string.digits))
admin_secret = ensure("s3_admin_secret_access_key", lambda: token(40))
api_key = ensure("s3_access_key_id", lambda: "API" + token(17, string.ascii_uppercase + string.digits))
api_secret = ensure("s3_secret_access_key", lambda: token(40))
bucket = bucket_name()
config = {
"identities": [
{
"name": "admin",
"credentials": [{"accessKey": admin_key, "secretKey": admin_secret}],
"actions": ["Admin", "Read", "Write", "List", "Tagging"],
},
{
"name": "messenger-api",
"credentials": [{"accessKey": api_key, "secretKey": api_secret}],
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"],
},
]
}
(SECRETS / "s3_config.json").write_bytes(json.dumps(config, indent=1).encode("utf-8"))
print(f"write secrets/s3_config.json (bucket {bucket}; identities admin, messenger-api; no anonymous)")
return 0
if __name__ == "__main__":
raise SystemExit(main())
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Peak RSS of `djpeg -strict -scale 1/8` on worst-case 40 Mpx JPEGs (sizing the API limit).
Run in a throwaway container with Pillow + libjpeg-turbo-progs:
python scripts/jpeg_decode_memory.py gen # writes /tmp/j/*.jpg (8000×5000)
python scripts/jpeg_decode_memory.py measure # one fresh child per file
The measuring parent stays tiny so RUSAGE_CHILDREN reflects djpeg only.
"""
import os
import resource
import subprocess
import sys
import time
OUT = "/tmp/j"
def gen() -> None:
from PIL import Image
os.makedirs(OUT, exist_ok=True)
noise = Image.effect_noise((8000, 5000), 64).convert("L")
rgb = Image.merge("RGB", (noise, noise.rotate(180), noise.transpose(Image.FLIP_LEFT_RIGHT)))
for name, img, kw in (
("baseline_420", rgb, dict(quality=30)),
("progressive_420", rgb, dict(quality=30, progressive=True)),
("progressive_444", rgb, dict(quality=20, progressive=True, subsampling=0)),
("progressive_cmyk", rgb.convert("CMYK"), dict(quality=15, progressive=True, subsampling=0)),
):
img.save(os.path.join(OUT, name + ".jpg"), "JPEG", **kw)
def measure_one(path: str) -> None:
t0 = time.monotonic()
proc = subprocess.run(["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path], capture_output=True)
took = time.monotonic() - t0
peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1024
print(f"{os.path.basename(path):22} {os.path.getsize(path) / 2**20:6.2f} MiB rc={proc.returncode} "
f"{took:5.2f}s djpeg_peak_rss={peak:.1f} MiB", flush=True)
if __name__ == "__main__":
if sys.argv[1] == "gen":
gen()
elif sys.argv[1] == "one":
measure_one(sys.argv[2])
else:
for name in sorted(os.listdir(OUT)):
subprocess.run([sys.executable, __file__, "one", os.path.join(OUT, name)], check=True)
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""LB balancing under mixed traffic: slow uploads interleaved with short GETs.
Each of 8 uploads (~1 MiB PNG) streams its body over ~4 s; between upload starts the
client sends one short GET. Records which replica served every upload and GET.
The balance algorithm is read from the running LB config. Results are merged into
evidence/lb-interleave.json keyed by algorithm. Tokens are kept in memory only.
"""
from __future__ import annotations
import base64
import collections
import http.client
import json
import os
import subprocess
import sys
import threading
import time
import uuid
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
HOST = os.environ.get("LB_HOST", "127.0.0.1")
PORT = int(os.environ.get("LB_PORT", "8080"))
UPLOADS = 8
def call(method, path, body=None, token=None, headers=None):
h = {"Accept": "application/json", **(headers or {})}
data = None
if body is not None:
data = json.dumps(body).encode()
h["Content-Type"] = "application/json"
if token:
h["Authorization"] = "Bearer " + token
req = urllib.request.Request(f"http://{HOST}:{PORT}{path}", data=data, headers=h, method=method)
resp = urllib.request.urlopen(req, timeout=30)
return resp.headers, json.loads(resp.read() or b"null")
def actor(label):
username, password = f"ilv_{label}_{uuid.uuid4().hex[:10]}", "T3st-" + uuid.uuid4().hex
_, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
_, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
return user["id"], session["token"]
def balance_algorithm():
cfg = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "exec", "-T", "lb",
"cat", "/usr/local/etc/haproxy/haproxy.cfg"], cwd=ROOT, text=True)
algos = {line.split()[1] for line in cfg.splitlines() if line.strip().startswith("balance ")}
return "/".join(sorted(algos))
def main():
algo = balance_algorithm()
alice, token = actor("a")
bob, _ = actor("b")
_, chat = call("POST", "/api/v1/chats", {"title": "interleave", "member_ids": [alice, bob]}, token=token)
payload = imagegen.rgb_png(700, 500, noise=True)
uploads, gets = [], []
def slow_upload(i):
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=ilv{i}.png")
for k, v in {"Authorization": "Bearer " + token, "Content-Type": "image/png",
"Content-Length": str(len(payload))}.items():
conn.putheader(k, v)
conn.endheaders()
step = len(payload) // 20
for off in range(0, len(payload), step):
conn.send(payload[off:off + step])
time.sleep(0.2)
resp = conn.getresponse()
resp.read()
uploads.append({"upload": i, "status": resp.status, "instance": resp.getheader("X-Instance-Id")})
threads = []
for i in range(UPLOADS):
th = threading.Thread(target=slow_upload, args=(i,))
th.start()
threads.append(th)
time.sleep(0.15)
gets.append(call("GET", "/api/v1/users/me", token=token)[0]["X-Instance-Id"])
time.sleep(0.15)
for th in threads:
th.join()
result = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"balance": algo,
"uploads": UPLOADS,
"upload_bytes": len(payload),
"uploads_per_replica": dict(collections.Counter(u["instance"] for u in uploads)),
"upload_statuses": dict(collections.Counter(str(u["status"]) for u in uploads)),
"short_gets_per_replica": dict(collections.Counter(gets)),
}
out = ROOT / "evidence" / "lb-interleave.json"
report = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {}
report[algo] = result
out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
print(json.dumps(result, ensure_ascii=False))
print("WROTE", out)
if __name__ == "__main__":
main()
+18
View File
@@ -0,0 +1,18 @@
@echo off
setlocal
rem Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); volumes are preserved.
cd /d "%~dp0\.."
if "%API_REPLICAS%"=="" set API_REPLICAS=2
docker compose --env-file .env restart postgres redis s3
if errorlevel 1 exit /b 1
rem s3-gw shares the s3 network namespace and must be restarted after it.
docker compose --env-file .env restart s3-gw
if errorlevel 1 exit /b 1
rem "up -d" alone leaves running replicas untouched; restart them, then restore the scale.
docker compose --env-file .env restart api
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps --scale api=%API_REPLICAS% api
if errorlevel 1 exit /b 1
docker compose --env-file .env restart lb
if errorlevel 1 exit /b 1
exit /b 0
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
# Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); preserve volumes.
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
docker compose --env-file .env restart postgres redis s3
# s3-gw shares the s3 network namespace and must be restarted after it.
docker compose --env-file .env restart s3-gw
# "up -d" alone leaves running replicas untouched; restart them, then restore the scale
# (migrate/s3-init already applied; skip one-shots).
docker compose --env-file .env restart api
docker compose --env-file .env up -d --no-deps --scale "api=${REPLICAS}" api
docker compose --env-file .env restart lb
exit 0
+51
View File
@@ -0,0 +1,51 @@
@echo off
setlocal EnableExtensions EnableDelayedExpansion
rem Grade-4 rolling replacement: replace API containers one-by-one under LB.
rem Usage: scripts\rolling-update.cmd
cd /d "%~dp0\.."
set "REPLICAS=%API_REPLICAS%"
if "%REPLICAS%"=="" set "REPLICAS=2"
set "URL=http://127.0.0.1:8080/health/ready"
set "IDS="
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do set "IDS=!IDS! %%I"
set "KNOWN=%IDS%"
set /a COUNT=0
for %%I in (%IDS%) do (
set /a COUNT+=1
echo rolling: replacing %%I
rem -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
docker stop -t 15 %%I >nul || exit /b 1
docker rm %%I >nul || exit /b 1
rem --no-recreate: the surviving replica keeps serving even if the image/config changed.
docker compose --env-file .env up -d --no-deps --no-recreate --scale api=%REPLICAS% api || exit /b 1
call :wait_new || exit /b 1
)
echo rolling update finished (%COUNT% steps, scale=%REPLICAS%)
exit /b 0
:wait_new
set "NEW="
for /f "usebackq delims=" %%J in (`docker compose --env-file .env ps -q api`) do (
if "!KNOWN:%%J=!"=="!KNOWN!" set "NEW=%%J"
)
if "%NEW%"=="" (
echo rolling: expected one new replica >&2
exit /b 1
)
set "KNOWN=%KNOWN% %NEW%"
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %NEW%`) do set "HOST=%%H"
rem Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
set /a TRIES=0
:poll
curl.exe -s -D - -o NUL "%URL%" | findstr /I /B /C:"x-instance-id: %HOST%" >nul && goto ready
set /a TRIES+=1
if %TRIES% geq 90 (
echo rolling: %HOST% not in LB rotation >&2
exit /b 1
)
ping -n 2 127.0.0.1 >nul
goto poll
:ready
echo rolling: step %COUNT% ready (%HOST%)
exit /b 0
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
set -eu
# Grade-4 rolling replacement: replace API containers one-by-one under LB.
# Usage: scripts/rolling-update.sh
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
URL="http://127.0.0.1:8080/health/ready"
ids=$(docker compose --env-file .env ps -q api | tr -d '\r')
known="$ids"
count=0
for id in $ids; do
[ -n "$id" ] || continue
count=$((count + 1))
printf '%s\n' "rolling: replacing $id"
# -t 15 = stop_grace_period: the replica drains ~3 s before it exits.
docker stop -t 15 "$id" >/dev/null
docker rm "$id" >/dev/null
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=${REPLICAS}" api
new=$(docker compose --env-file .env ps -q api | tr -d '\r' | grep -v -F "$known" || true)
[ "$(printf '%s\n' "$new" | grep -c .)" -eq 1 ] || { printf '%s\n' "rolling: expected one new replica" >&2; exit 1; }
known="$known
$new"
host=$(docker inspect -f '{{.Config.Hostname}}' "$new" | tr -d '\r')
# Public ready is 200 while the other replica serves; wait until the LB routes to the new one.
i=0
until curl -s -D - -o /dev/null "$URL" | tr -d '\r' | grep -qi "^x-instance-id: ${host}\$"; do
i=$((i + 1))
[ "$i" -lt 180 ] || { printf '%s\n' "rolling: $host not in LB rotation" >&2; exit 1; }
sleep 0.5
done
printf '%s\n' "rolling: step $count ready ($host)"
done
printf '%s\n' "rolling update finished ($count steps, scale=${REPLICAS})"
+170
View File
@@ -0,0 +1,170 @@
#!/usr/bin/env python3
"""Grade-4: closed-loop POSTs during rolling API replacement."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import threading
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
REPLICAS = os.environ.get("API_REPLICAS", "2")
def call(method, path, body=None, token=None, headers=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
except Exception as exc:
return {"status": 0, "error": type(exc).__name__}
raw = resp.read()
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")}
def api_ids():
out = subprocess.check_output(
["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True
)
return out.split()
def hostname(cid):
return subprocess.check_output(
["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True
).strip()
def replace_replica(old_id, known):
"""Stop+remove one replica, create a fresh one, wait until the LB routes to it."""
subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL)
subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL)
# --no-recreate: the surviving replica keeps serving even if the image/config changed.
subprocess.check_call(
["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate",
"--scale", f"api={REPLICAS}", "api"],
cwd=ROOT,
)
new = [cid for cid in api_ids() if cid not in known]
if len(new) != 1:
raise RuntimeError(f"expected one new replica, got {new}")
host = hostname(new[0])
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
r = call("GET", "/health/ready")
if r.get("status") == 200 and r.get("instance") == host:
return new[0], host
time.sleep(0.5)
raise RuntimeError(f"new replica {host} not in LB rotation after 90s")
def bootstrap():
uname = "roll_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw})
assert user["status"] == 201, user
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"]
peer_u = "rollb_" + uuid.uuid4().hex[:10]
peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]},
token=tok,
)["json"]
return tok, chat["id"]
def main():
tok, chat_id = bootstrap()
stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []}
stop = threading.Event()
def worker():
while not stop.is_set():
key = uuid.uuid4().hex
t0 = time.monotonic()
r = call(
"POST",
f"/api/v1/chats/{chat_id}/messages",
{"text": "roll-" + key},
token=tok,
headers={"Idempotency-Key": key},
)
dt = (time.monotonic() - t0) * 1000
if r.get("status") == 201:
stats["ok"] += 1
stats["latencies_ms"].append(dt)
else:
stats["err"] += 1
msg = ((r.get("json") or {}).get("error") or {}).get("message", "")
code = f"{r.get('status') or r.get('error')} {msg}".strip()
stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1
time.sleep(0.05)
ids = api_ids()
known = set(ids)
replaced = []
threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)]
started = time.monotonic()
for th in threads:
th.start()
time.sleep(2) # warm-up
# Rolling: replace each original replica in turn; the next one is stopped
# only after the LB already routes requests to the fresh container.
for cid in ids:
old_host = hostname(cid)
new_id, new_host = replace_replica(cid, known)
known.add(new_id)
replaced.append({"old": old_host, "new": new_host})
time.sleep(2)
time.sleep(2)
stop.set()
for th in threads:
th.join(timeout=5)
duration = time.monotonic() - started
lats = sorted(stats["latencies_ms"])
def pct(p):
if not lats:
return None
return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2)
summary = {
"successful": stats["ok"],
"errors": stats["err"],
"errors_by_status": stats["err_by_status"],
"p50_ms": pct(0.50),
"p95_ms": pct(0.95),
"p99_ms": pct(0.99),
"duration_s": round(duration, 1),
"replicas_rolled": len(replaced),
"replaced": replaced,
}
out = ROOT / "evidence" / "rolling-load.json"
out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
print(json.dumps(summary, ensure_ascii=False))
print("WROTE", out)
if __name__ == "__main__":
main()
+171
View File
@@ -0,0 +1,171 @@
#!/usr/bin/env python3
"""Grade-4 evidence: stop each observed API replica in turn; restore between steps."""
from __future__ import annotations
import base64
import json
import os
import subprocess
import time
import uuid
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
def call(method, path, body=None, token=None, headers=None, expected=None):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
data = None
if body is not None:
data = json.dumps(body).encode("utf-8")
h["Content-Type"] = "application/json"
if headers:
h.update(headers)
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=5)
except urllib.error.HTTPError as err:
resp = err
raw = resp.read()
status = resp.code
if expected is not None and status != expected:
raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}")
js = None
if raw and resp.headers.get_content_type() == "application/json":
js = json.loads(raw.decode("utf-8"))
return {"status": status, "headers": resp.headers, "json": js}
def header(resp, name: str) -> str:
return resp["headers"].get(name) or resp["headers"].get(name.lower()) or ""
def restore_api(target: str):
replicas = os.environ.get("API_REPLICAS", "2")
subprocess.check_call(
[
"docker",
"compose",
"--env-file",
".env",
"up",
"-d",
"--no-deps",
"--scale",
f"api={replicas}",
"--remove-orphans",
"api",
],
cwd=ROOT,
)
# Public ready is 200 via the surviving replica; wait until the LB routes to the restored one.
for _ in range(180):
try:
if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target:
return
except Exception:
pass
time.sleep(0.5)
raise RuntimeError(f"replica {target} not back in LB rotation after restore")
def one_round(round_no: int) -> dict:
uname = "seq_" + uuid.uuid4().hex[:10]
pw = "T3st-" + uuid.uuid4().hex
user = call(
"POST",
"/api/v1/users",
{"username": uname, "display_name": "Seq", "password": pw},
expected=201,
)["json"]
cred = base64.b64encode(f"{uname}:{pw}".encode()).decode()
tok = call(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": "Basic " + cred},
expected=201,
)["json"]["token"]
peer_u = "seqb_" + uuid.uuid4().hex[:10]
peer = call(
"POST",
"/api/v1/users",
{"username": peer_u, "display_name": "SeqB", "password": pw},
expected=201,
)["json"]
chat = call(
"POST",
"/api/v1/chats",
{"title": "seq", "member_ids": [user["id"], peer["id"]]},
token=tok,
expected=201,
)["json"]
observed = set()
for _ in range(40):
observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id"))
if len(observed) < 2:
raise RuntimeError(f"need ≥2 instances, got {observed}")
target = sorted(observed)[round_no % len(observed)]
env = dict(os.environ)
env["TARGET_INSTANCE_ID"] = target
script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh")
started = time.monotonic()
proc = subprocess.Popen([str(script)], env=env, cwd=ROOT)
ok = fail = 0
first = None
after = []
while time.monotonic() - started < 20:
code = proc.poll()
try:
key = uuid.uuid4().hex
r = call(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
{"text": "seq-" + key},
token=tok,
headers={"Idempotency-Key": key},
expected=201,
)
ok += 1
if code == 0:
if first is None:
first = time.monotonic() - started
after.append(header(r, "X-Instance-Id"))
except Exception:
fail += 1
time.sleep(0.1)
if proc.poll() is None:
proc.wait(timeout=10)
result = {
"round": round_no,
"observed_before": sorted(observed),
"target": target,
"successful_writes": ok,
"failed_attempts": fail,
"first_recovery_seconds": round(first, 3) if first is not None else None,
"target_absent_in_last5": target not in after[-5:],
}
t0 = time.monotonic()
restore_api(target)
result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3)
return result
def main():
results = []
for i in range(2):
results.append(one_round(i))
print(json.dumps(results[-1], ensure_ascii=False))
out = ROOT / "evidence" / "seq-failover.json"
out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
print("WROTE", out)
if __name__ == "__main__":
main()
+17
View File
@@ -0,0 +1,17 @@
@echo off
setlocal EnableExtensions
cd /d "%~dp0\.."
if "%TARGET_INSTANCE_ID%"=="" (
echo stop-one: TARGET_INSTANCE_ID required >&2
exit /b 1
)
for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do (
for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %%I`) do (
if /I "%%H"=="%TARGET_INSTANCE_ID%" (
docker stop -t 15 %%I >nul
exit /b 0
)
)
)
echo stop-one: replica not found >&2
exit /b 1
+44
View File
@@ -0,0 +1,44 @@
#!/bin/sh
set -eu
# Stop ONE API replica matching TARGET_INSTANCE_ID (= X-Instance-Id / hostname).
# Leave it stopped; do not touch DB/Redis/LB/volumes.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
found=""
# Compose project containers for service api
ids=$(docker compose --env-file .env ps -q api 2>/dev/null || true)
for id in $ids; do
[ -n "$id" ] || continue
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
iid=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$id" | sed -n 's/^INSTANCE_ID=//p' | head -n1)
if [ -n "$iid" ] && [ "$iid" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
if [ -z "$found" ]; then
# Fallback: any running container with matching hostname (scaled names differ).
for id in $(docker ps -q --filter "ancestor=mtusi-messenger:lab4"); do
host=$(docker inspect -f '{{.Config.Hostname}}' "$id")
if [ "$host" = "$TARGET_INSTANCE_ID" ]; then
found=$id
break
fi
done
fi
if [ -z "$found" ]; then
printf '%s\n' "stop-one: replica not found for TARGET_INSTANCE_ID" >&2
exit 1
fi
# Graceful stop (= stop_grace_period): on SIGTERM the API reports not-ready but keeps
# serving for a 3 s drain window, then finishes in-flight requests and exits (~3.5 s).
docker stop -t 15 "$found" >/dev/null
exit 0
+12
View File
@@ -0,0 +1,12 @@
@echo off
setlocal
cd /d "%~dp0\.."
if "%API_REPLICAS%"=="" set API_REPLICAS=2
python scripts\init_secrets.py
if errorlevel 1 exit /b 1
docker compose --env-file .env config --quiet
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --build --scale api=%API_REPLICAS%
if errorlevel 1 exit /b 1
echo stack up: api scale=%API_REPLICAS%, entry http://127.0.0.1:8080
exit /b 0
Executable
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
# Bootstrap lab4 stack with ≥2 API replicas behind LB, PostgreSQL, Redis and private S3.
cd "$(dirname "$0")/.."
REPLICAS="${API_REPLICAS:-2}"
python3 scripts/init_secrets.py
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale "api=${REPLICAS}"
printf '%s\n' "stack up: api scale=${REPLICAS}, entry http://127.0.0.1:8080"
+284
View File
@@ -0,0 +1,284 @@
#!/usr/bin/env python3
"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads.
Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from
file) keeps the API process RSS growth far below N × file size; total in-flight
work per replica is capped by MAX_CONCURRENT_UPLOADS.
Method (per concurrency level, fresh replicas):
1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready;
2. one warm-up upload per replica (imports, pools, TLS-less S3 connections);
3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline;
4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat
(anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s;
5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak.
Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5]
Writes evidence/upload-memory.json. Tokens are kept in memory only.
"""
from __future__ import annotations
import argparse
import base64
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
import imagegen # noqa: E402
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def sh(args, check=True) -> str:
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}")
return proc.stdout
def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180):
h = {"Accept": "application/json"}
if token:
h["Authorization"] = "Bearer " + token
if body is not None:
raw = json.dumps(body).encode()
h["Content-Type"] = "application/json"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method)
try:
resp = urllib.request.urlopen(req, timeout=timeout)
except urllib.error.HTTPError as err:
resp = err
data = resp.read()
return resp.code, resp.headers, data
def actor(label):
username = f"mem_{label}_{uuid.uuid4().hex[:10]}"
password = "T3st-" + uuid.uuid4().hex
status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
assert status == 201, status
user = json.loads(data)
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
assert status == 201, status
return user["id"], json.loads(data)["token"]
def wait_ready(timeout=120):
deadline = time.time() + timeout
while time.time() < deadline:
try:
if request("GET", "/health/ready", timeout=3)[0] == 200:
seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)}
if len(seen) >= 2:
return
except OSError:
pass
time.sleep(1)
raise RuntimeError("stack not ready")
def containers():
ids = sh(COMPOSE + ["ps", "-q", "api"]).split()
return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids}
def proc_status(cid):
out = sh(["docker", "exec", cid, "cat", "/proc/1/status"])
fields = {}
for line in out.splitlines():
if line.startswith(("VmRSS", "VmHWM")):
k, v = line.split(":", 1)
fields[k] = int(v.split()[0]) // 1024 # MiB
return fields
def cgroup(cid):
stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"])
vals = dict(line.split() for line in stat.splitlines() if line)
peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip()
current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip()
mib = 1024 * 1024
return {
"anon_mib": round(int(vals["anon"]) / mib, 1),
"file_mib": round(int(vals["file"]) / mib, 1),
"current_mib": round(int(current) / mib, 1),
"peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None,
}
def sampler(cid, stop, out):
best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0}
while not stop.is_set():
try:
c = cgroup(cid)
for k in ("anon_mib", "file_mib", "current_mib"):
best[k] = max(best[k], c[k])
best["samples"] += 1
except Exception:
pass
time.sleep(0.25)
out[cid] = best
def pct(values, q):
values = sorted(values)
return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3)
def run_level(n, payload, chat_id, token):
print(f"== level {n}: restarting s3, api replicas and lb", flush=True)
# s3-gw shares the s3 network namespace and must be restarted after it.
sh(COMPOSE + ["restart", "s3"])
sh(COMPOSE + ["restart", "s3-gw", "api", "lb"])
wait_ready()
by_host = containers()
lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0]
s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0]
# Warm-up: at least one upload served by every replica.
served = set()
for _ in range(20):
_, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token,
raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"})
served.add(h.get("X-Instance-Id"))
if served >= set(by_host):
break
baseline = {}
for host, cid in by_host.items():
sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"])
baseline[host] = {**proc_status(cid), **cgroup(cid)}
lb_baseline = cgroup(lb)
s3_baseline = cgroup(s3c)
stop = threading.Event()
sampled = {}
threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]]
for t in threads:
t.start()
time.sleep(0.5)
barrier = threading.Barrier(n)
results = [None] * n
def one(i):
barrier.wait()
t0 = time.monotonic()
try:
status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png",
token=token, raw=payload, headers={"Content-Type": "image/png"})
results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id"))
except OSError as exc:
results[i] = (type(exc).__name__, time.monotonic() - t0, None)
workers = [threading.Thread(target=one, args=(i,)) for i in range(n)]
started = time.monotonic()
for w in workers:
w.start()
for w in workers:
w.join()
wall = time.monotonic() - started
time.sleep(1.0)
stop.set()
for t in threads:
t.join()
replicas = {}
for host, cid in by_host.items():
after = proc_status(cid)
cg = cgroup(cid)
replicas[host] = {
"uploads_served": sum(1 for r in results if r[2] == host),
"rss_baseline_mib": baseline[host]["VmRSS"],
"rss_peak_vmhwm_mib": after["VmHWM"],
"rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"],
"cgroup_peak_mib": cg["peak_mib"],
"cgroup_baseline_current_mib": baseline[host]["current_mib"],
"sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"),
"sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"),
"samples": sampled.get(cid, {}).get("samples"),
}
latencies = [r[1] for r in results if r[0] == 201]
statuses = {}
for r in results:
statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1
per_replica_max = max(v["uploads_served"] for v in replicas.values())
return {
"concurrency": n,
"payload_bytes": len(payload),
"statuses": statuses,
"wall_seconds": round(wall, 3),
"latency_ok_p50_s": pct(latencies, 0.5) if latencies else None,
"latency_ok_p95_s": pct(latencies, 0.95) if latencies else None,
"latency_ok_max_s": round(max(latencies), 3) if latencies else None,
"throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1),
"naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1),
"replicas": replicas,
"lb": {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024),
"baseline_current_mib": lb_baseline["current_mib"],
"cgroup_peak_mib": cgroup(lb)["peak_mib"],
"sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"),
"sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"),
"restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()),
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true",
},
"s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})),
}
def _s3_report(cid, baseline, sampled):
running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true"
return {
"mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024),
"baseline_current_mib": baseline["current_mib"],
# memory.peak includes reclaimable page cache of the volume files; anon is the weed heap.
"cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None,
"sampled_max_anon_mib": sampled.get("anon_mib"),
"sampled_max_current_mib": sampled.get("current_mib"),
"running": running,
"oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true",
}
def main():
p = argparse.ArgumentParser(description="peak memory under concurrent uploads")
p.add_argument("--levels", type=int, nargs="+", default=[10, 20])
p.add_argument("--size-mib", type=float, default=9.5)
p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json"))
args = p.parse_args()
payload = imagegen.large_png(int(args.size_mib * 1024 * 1024))
_, token = actor("a")
bob_id, _ = actor("b")
me = json.loads(request("GET", "/api/v1/users/me", token=token)[2])
status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token)
assert status == 201, status
chat_id = json.loads(data)["id"]
limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}",
list(containers().values())[0]]).split()
report = {
"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
"api_mem_limit_mib": int(limits[0]) // (1024 * 1024),
"api_cpus": int(limits[1]) / 1e9,
"max_concurrent_uploads_per_replica": 8,
"levels": [],
}
for n in args.levels:
level = run_level(n, payload, chat_id, token)
report["levels"].append(level)
print(json.dumps(level, ensure_ascii=False), flush=True)
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8")
print("WROTE", args.output)
if __name__ == "__main__":
main()