forked from mtusi2026/mtusi2026lab4
lab4: память загрузок и S3 под лимитом, документация защиты
- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg; - PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT); - djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415; - api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты одновременных загрузок SeaweedFS, память 1g; - docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4; scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4; - bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
a2d867bec6
commit
4ea6924db1
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
|
||||
|
||||
python scripts/collect_evidence.py # ~25 min; the stand must NOT be running yet
|
||||
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
|
||||
|
||||
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
|
||||
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
|
||||
level 4 experiments, and records container states before/after every hook. Prints no secrets
|
||||
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
|
||||
container states before/after every hook. Prints no secrets
|
||||
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
|
||||
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
|
||||
"""
|
||||
@@ -95,6 +96,7 @@ def main() -> int:
|
||||
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
|
||||
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
# venv for validate / own tests
|
||||
if not Path(VENV_PY).exists():
|
||||
@@ -109,11 +111,11 @@ def main() -> int:
|
||||
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
|
||||
|
||||
# 3. persistence with states before/after
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey")
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey; volumes kept):\n" + "\n".join(states()))
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
|
||||
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
|
||||
@@ -147,10 +149,11 @@ def main() -> int:
|
||||
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
|
||||
timeout=3600)
|
||||
|
||||
# 6. level 4 experiments and the public load script
|
||||
log = Log("level4-runs.txt", "Level 4: failover series, rolling update under load, public load.py")
|
||||
log.run([PYTHON, "bench/failover_series.py", "--output", "evidence/failover"], timeout=1800)
|
||||
log.run([PYTHON, "bench/rolling_under_load.py", "--output", "evidence/rolling-update"], timeout=1800)
|
||||
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
|
||||
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
|
||||
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
|
||||
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
|
||||
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
|
||||
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
||||
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
||||
|
||||
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
PASSWORD = "Demo-password-Ω-12345"
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base: str):
|
||||
self.base = base.rstrip("/")
|
||||
|
||||
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
|
||||
headers: dict | None = None) -> tuple[int, dict, bytes]:
|
||||
h = dict(headers or {})
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, dict(r.headers), r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, dict(e.headers), e.read()
|
||||
|
||||
|
||||
def show(label: str, status: int, headers: dict, payload: bytes, *, binary: bool = False) -> None:
|
||||
inst = headers.get("X-Instance-Id", "-")
|
||||
if binary:
|
||||
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
|
||||
else:
|
||||
text = payload.decode("utf-8", "replace")[:300]
|
||||
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
|
||||
|
||||
|
||||
def user(c: Client, label: str) -> dict:
|
||||
name = f"{label}_{uuid.uuid4().hex[:8]}"
|
||||
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
|
||||
"password": PASSWORD})
|
||||
assert status == 201, body
|
||||
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
||||
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
session = json.loads(body)
|
||||
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
|
||||
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
||||
print(f"users: alice, bob (members), eve (outsider)")
|
||||
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
|
||||
print("\n1. upload -> message -> download by the second member")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
|
||||
att = json.loads(b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id} (bob)", st, h, b)
|
||||
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
||||
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
||||
print(f" SHA-256 of the download == fixture == metadata: {same}")
|
||||
|
||||
print("\n2. outsider and direct S3 access")
|
||||
for suffix in ("", "/content"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
|
||||
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
|
||||
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||||
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
|
||||
f"{key not in json.dumps(att)})")
|
||||
probe = (
|
||||
"import urllib.request as u, urllib.error as e, asyncio\n"
|
||||
"def st(url):\n"
|
||||
" try: return u.urlopen(url, timeout=3).status\n"
|
||||
" except e.HTTPError as x: return x.code\n"
|
||||
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
|
||||
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
||||
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
||||
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
|
||||
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
print(" (inside network storage) " + line)
|
||||
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
|
||||
return 0 if same else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user