lab4: память загрузок и S3 под лимитом, документация защиты

- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg;
- PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT);
- djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415;
- api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты
  одновременных загрузок SeaweedFS, память 1g;
- docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4;
  scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4;
- bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Alexey Bodanyuk
2026-10-25 14:21:43 +03:00
co-authored by Claude Opus 5.5
parent a2d867bec6
commit 4ea6924db1
18 changed files with 701 additions and 120 deletions
+11 -8
View File
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
python scripts/collect_evidence.py # ~25 min; the stand must NOT be running yet
python scripts/collect_evidence.py # ~60 min; the stand must NOT be running yet
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
stand, then runs make check/validate/test/test-persistence/test-failover, the own tests, the
level 4 experiments, and records container states before/after every hook. Prints no secrets
lab 4 experiments (memory under concurrent uploads, reconciler faults and cost), and records
container states before/after every hook. Prints no secrets
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
"""
@@ -95,6 +96,7 @@ def main() -> int:
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
# venv for validate / own tests
if not Path(VENV_PY).exists():
@@ -109,11 +111,11 @@ def main() -> int:
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
# 3. persistence with states before/after
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey")
log = Log("test-persistence.txt", "make test-persistence: restart of LB, all API replicas, PostgreSQL, Valkey, S3")
log.note("Containers BEFORE:\n" + "\n".join(states()))
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey; volumes kept):\n" + "\n".join(states()))
log.note("Containers AFTER (new pid/started for lb, api-*, postgres, valkey, s3; volumes kept):\n" + "\n".join(states()))
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
@@ -147,10 +149,11 @@ def main() -> int:
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
timeout=3600)
# 6. level 4 experiments and the public load script
log = Log("level4-runs.txt", "Level 4: failover series, rolling update under load, public load.py")
log.run([PYTHON, "bench/failover_series.py", "--output", "evidence/failover"], timeout=1800)
log.run([PYTHON, "bench/rolling_under_load.py", "--output", "evidence/rolling-update"], timeout=1800)
# 6. lab 4 experiments (level 4 memory, track 2 reconciler) and the public load script
log = Log("lab4-runs.txt", "Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py")
log.run([VENV_PY, "bench/upload_memory.py", "--output", "evidence/memory"], timeout=3600)
log.run([VENV_PY, "bench/reconcile_faults.py", "--output", "evidence/reconcile"], timeout=1800)
log.run([VENV_PY, "bench/reconcile_cost.py", "--output", "evidence/reconcile"], timeout=3600)
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
request straight to S3 (from inside the storage network) is refused, the signed one works.
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
"""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import subprocess
import sys
import urllib.error
import urllib.request
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
PASSWORD = "Demo-password-Ω-12345"
class Client:
def __init__(self, base: str):
self.base = base.rstrip("/")
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
headers: dict | None = None) -> tuple[int, dict, bytes]:
h = dict(headers or {})
if token:
h["Authorization"] = "Bearer " + token
data = raw
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
h["Content-Type"] = "application/json"
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
try:
with urllib.request.urlopen(req, timeout=30) as r:
return r.status, dict(r.headers), r.read()
except urllib.error.HTTPError as e:
return e.code, dict(e.headers), e.read()
def show(label: str, status: int, headers: dict, payload: bytes, *, binary: bool = False) -> None:
inst = headers.get("X-Instance-Id", "-")
if binary:
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
else:
text = payload.decode("utf-8", "replace")[:300]
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
def user(c: Client, label: str) -> dict:
name = f"{label}_{uuid.uuid4().hex[:8]}"
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
"password": PASSWORD})
assert status == 201, body
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
session = json.loads(body)
return {"id": session["user"]["id"], "token": session["token"], "name": name}
def main() -> int:
p = argparse.ArgumentParser()
p.add_argument("--base-url", default="http://127.0.0.1:8080")
args = p.parse_args()
c = Client(args.base_url)
content = FIXTURE.read_bytes()
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
print(f"users: alice, bob (members), eve (outsider)")
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
chat = json.loads(body)
print("\n1. upload -> message -> download by the second member")
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
raw=content, headers={"Content-Type": "image/png"})
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
att = json.loads(b)
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
show("GET /attachments/{id} (bob)", st, h, b)
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
print(f" SHA-256 of the download == fixture == metadata: {same}")
print("\n2. outsider and direct S3 access")
for suffix in ("", "/content"):
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
f"{key not in json.dumps(att)})")
probe = (
"import urllib.request as u, urllib.error as e, asyncio\n"
"def st(url):\n"
" try: return u.urlopen(url, timeout=3).status\n"
" except e.HTTPError as x: return x.code\n"
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
"asyncio.run(m())\n")
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
capture_output=True, text=True).stdout
for line in out.splitlines():
print(" (inside network storage) " + line)
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
return 0 if same else 1
if __name__ == "__main__":
sys.exit(main())