forked from mtusi2026/mtusi2026lab4
147 lines
7.7 KiB
Python
147 lines
7.7 KiB
Python
"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен.
|
|
|
|
Нужен запущенный стенд из compose.yaml.
|
|
"""
|
|
import json
|
|
import socket
|
|
|
|
import images
|
|
from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload
|
|
|
|
# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API.
|
|
ANONYMOUS = r'''
|
|
import json, sys, urllib.request, urllib.error
|
|
import settings
|
|
base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET
|
|
def status(method, url, data=None):
|
|
try:
|
|
with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r:
|
|
return r.status, r.read(300).decode("utf-8", "replace")
|
|
except urllib.error.HTTPError as e:
|
|
return e.code, e.read(300).decode("utf-8", "replace")
|
|
out = {}
|
|
for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None),
|
|
("head_object", "HEAD", base + "/" + sys.argv[1], None),
|
|
("list_bucket", "GET", base + "?list-type=2", None),
|
|
("put_object", "PUT", base + "/originals/anonymous-test", b"x"),
|
|
("get_bucket_policy", "GET", base + "?policy", None)):
|
|
code, body = status(method, url, data)
|
|
out[name] = {"status": code, "s3_code": body.split("<Code>")[1].split("</Code>")[0] if "<Code>" in body else ""}
|
|
print(json.dumps(out))
|
|
'''
|
|
|
|
# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json).
|
|
API_KEY_RIGHTS = r'''
|
|
import io, json, sys, uuid
|
|
from botocore.exceptions import ClientError
|
|
import settings, storage
|
|
c, bucket = storage._client, settings.S3_BUCKET
|
|
def attempt(name, fn):
|
|
try:
|
|
fn(); return "allowed"
|
|
except ClientError as e:
|
|
return "denied: " + e.response.get("Error", {}).get("Code", "?")
|
|
public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]},
|
|
"Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
|
|
key = "originals/" + uuid.uuid4().hex
|
|
result = {
|
|
"get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()),
|
|
"put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")),
|
|
"delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)),
|
|
"list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)),
|
|
"create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])),
|
|
"make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)),
|
|
"read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)),
|
|
"delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)),
|
|
"change versioning": attempt("versioning", lambda: c.put_bucket_versioning(
|
|
Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})),
|
|
}
|
|
try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket
|
|
result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", []))
|
|
except ClientError as e:
|
|
result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?")
|
|
print(json.dumps(result))
|
|
'''
|
|
|
|
|
|
def last_json(output):
|
|
return json.loads(output.splitlines()[-1])
|
|
|
|
|
|
def prepare():
|
|
alice, bob = new_user("alice"), new_user("bob")
|
|
chat = new_chat(alice, bob)
|
|
att = upload(alice, chat["id"], images.sample_png()).json
|
|
key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")
|
|
return alice, bob, att, key
|
|
|
|
|
|
def test_bucket_is_private():
|
|
alice, bob, att, key = prepare()
|
|
result = last_json(app_python(ANONYMOUS, key))
|
|
for name, reply in result.items():
|
|
print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}")
|
|
check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}")
|
|
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
|
print("the same file through the API for a chat member -> 200")
|
|
print("PASS: the bucket is private: no anonymous read, list, write or policy read")
|
|
|
|
|
|
def test_api_key_is_limited():
|
|
alice, bob, att, key = prepare()
|
|
result = last_json(app_python(API_KEY_RIGHTS, key))
|
|
allowed = {"get own object", "put object", "delete object", "list bucket"}
|
|
visible = result.pop("visible buckets")
|
|
print(f"API key: {'list buckets':<32} -> {visible}")
|
|
check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets")
|
|
for name, outcome in result.items():
|
|
print(f"API key: {name:<32} -> {outcome}")
|
|
if name in allowed:
|
|
check(outcome == "allowed", f"API key must be allowed to {name}")
|
|
else:
|
|
check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}")
|
|
print("PASS: the API key can only get/put/delete/list objects of its own bucket; "
|
|
"bucket admin, policy and other buckets are denied")
|
|
|
|
|
|
def test_no_open_ports_or_console():
|
|
config = json.loads(compose_output("config", "--format", "json"))
|
|
s3 = config["services"]["s3"]
|
|
check(not s3.get("ports"), "s3 must not publish ports")
|
|
check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network")
|
|
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
|
|
print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, "
|
|
"filer UI 8888 are reachable only inside the stand)")
|
|
for port in (8333, 9333, 8888, 19333):
|
|
sock = socket.socket()
|
|
sock.settimeout(1)
|
|
refused = sock.connect_ex(("127.0.0.1", port)) != 0
|
|
sock.close()
|
|
check(refused, f"127.0.0.1:{port} is reachable from the host")
|
|
print(f"127.0.0.1:{port} from the host -> connection refused")
|
|
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()}
|
|
for name, items in sorted(secrets.items()):
|
|
print(f"secrets of {name}: {items or '-'}")
|
|
for name in ("api", "janitor"):
|
|
check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"},
|
|
f"{name} must not get S3 admin keys or the key config")
|
|
check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key")
|
|
print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init")
|
|
|
|
|
|
def test_bootstrap_is_idempotent():
|
|
alice, bob, att, key = prepare()
|
|
output = compose_output("run", "--rm", "--no-deps", "s3-init")
|
|
print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip()))
|
|
check("bucket is private" in output, "bootstrap did not confirm a private bucket")
|
|
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
|
check(got.data == images.sample_png(), "file changed after repeated bootstrap")
|
|
upload(alice, att["chat_id"], images.sample_png())
|
|
print("after the repeated bootstrap: old file downloads, new upload works")
|
|
print("PASS: bucket/user/policy bootstrap can be repeated safely")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console,
|
|
test_bootstrap_is_idempotent])
|