- тело загрузки: один буфер ровно Content-Length, без копий до S3 и djpeg; - PNG: zlib получает вход кусками по 64 KiB (unconsumed_tail копировал остаток IDAT); - djpeg, убитый OOM-killer'ом, -> 503 (ресурсы сервера), а не 415; - api: memswap_limit = memory (жёсткий лимит без swap); S3: GOMEMLIMIT и лимиты одновременных загрузок SeaweedFS, память 1g; - docs: CONSISTENCY.md, DEFENSE.md, WINDOWS.md, DATA_MODEL.md под lab4; scripts/demo_attachments.py для сценария защиты; CI и collect_evidence под lab4; - bench/reconcile_faults.py: отчёты трека 2 (ошибки с обеих сторон, прерванный apply). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
127 lines
6.2 KiB
Python
127 lines
6.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Defense scenario, steps 1-2 (README): upload the fixture, send a message with attachment_ids,
|
|
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
|
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
|
|
|
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
|
|
|
|
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
|
PASSWORD = "Demo-password-Ω-12345"
|
|
|
|
|
|
class Client:
|
|
def __init__(self, base: str):
|
|
self.base = base.rstrip("/")
|
|
|
|
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
|
|
headers: dict | None = None) -> tuple[int, dict, bytes]:
|
|
h = dict(headers or {})
|
|
if token:
|
|
h["Authorization"] = "Bearer " + token
|
|
data = raw
|
|
if body is not None:
|
|
data = json.dumps(body, ensure_ascii=False).encode()
|
|
h["Content-Type"] = "application/json"
|
|
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return r.status, dict(r.headers), r.read()
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, dict(e.headers), e.read()
|
|
|
|
|
|
def show(label: str, status: int, headers: dict, payload: bytes, *, binary: bool = False) -> None:
|
|
inst = headers.get("X-Instance-Id", "-")
|
|
if binary:
|
|
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
|
|
else:
|
|
text = payload.decode("utf-8", "replace")[:300]
|
|
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
|
|
|
|
|
|
def user(c: Client, label: str) -> dict:
|
|
name = f"{label}_{uuid.uuid4().hex[:8]}"
|
|
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
|
|
"password": PASSWORD})
|
|
assert status == 201, body
|
|
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
|
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
|
session = json.loads(body)
|
|
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
|
args = p.parse_args()
|
|
c = Client(args.base_url)
|
|
content = FIXTURE.read_bytes()
|
|
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
|
|
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
|
print(f"users: alice, bob (members), eve (outsider)")
|
|
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
|
body={"title": "Демо lab4", "member_ids": [alice["id"], bob["id"]]})
|
|
chat = json.loads(body)
|
|
|
|
print("\n1. upload -> message -> download by the second member")
|
|
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
|
raw=content, headers={"Content-Type": "image/png"})
|
|
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
|
|
att = json.loads(b)
|
|
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
|
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
|
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
|
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
|
show("GET /attachments/{id} (bob)", st, h, b)
|
|
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
|
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
|
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
|
print(f" SHA-256 of the download == fixture == metadata: {same}")
|
|
|
|
print("\n2. outsider and direct S3 access")
|
|
for suffix in ("", "/content"):
|
|
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
|
|
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
|
|
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
|
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
|
|
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
|
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
|
|
f"{key not in json.dumps(att)})")
|
|
probe = (
|
|
"import urllib.request as u, urllib.error as e, asyncio\n"
|
|
"def st(url):\n"
|
|
" try: return u.urlopen(url, timeout=3).status\n"
|
|
" except e.HTTPError as x: return x.code\n"
|
|
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
|
|
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
|
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
|
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
|
|
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
|
|
"asyncio.run(m())\n")
|
|
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
|
|
capture_output=True, text=True).stdout
|
|
for line in out.splitlines():
|
|
print(" (inside network storage) " + line)
|
|
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
|
|
return 0 if same else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|