Files
mtusi2026lab7/student-tests/test_s3_security.py
T

155 lines
8.6 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен.
lab7: S3 только по HTTPS (проверка учебного CA), у api/worker/janitor свои ключи с правами по prefix (из OpenBao);
admin-ключ — только у s3-init (тоже из OpenBao), в файлах контейнеров приложения его нет.
Нужен запущенный стенд из compose.yaml.
"""
import json
import socket
import images
from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload
# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API.
ANONYMOUS = r'''
import json, sys, urllib.request, urllib.error
import settings, tlsutil
base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}),
urllib.request.HTTPSHandler(context=tlsutil.client_context(cert=False)))
def status(method, url, data=None):
try:
with opener.open(urllib.request.Request(url, data=data, method=method), timeout=5) as r:
return r.status, r.read(300).decode("utf-8", "replace")
except urllib.error.HTTPError as e:
return e.code, e.read(300).decode("utf-8", "replace")
out = {}
for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None),
("head_object", "HEAD", base + "/" + sys.argv[1], None),
("list_bucket", "GET", base + "?list-type=2", None),
("put_object", "PUT", base + "/originals/anonymous-test", b"x"),
("get_bucket_policy", "GET", base + "?policy", None)):
code, body = status(method, url, data)
out[name] = {"status": code, "s3_code": body.split("<Code>")[1].split("</Code>")[0] if "<Code>" in body else ""}
print(json.dumps(out))
'''
# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json).
API_KEY_RIGHTS = r'''
import io, json, sys, uuid
from botocore.exceptions import ClientError
import settings, storage
c, bucket = storage._clients.get(), settings.S3_BUCKET
def attempt(name, fn):
try:
fn(); return "allowed"
except ClientError as e:
return "denied: " + e.response.get("Error", {}).get("Code", "?")
public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]},
"Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]})
key = "staging/" + uuid.uuid4().hex # lab7: api пишет только в staging/
result = {
"get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()),
"put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")),
"delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)),
"put into attachments/ (worker's prefix)": attempt("put2", lambda: c.put_object(
Bucket=bucket, Key="attachments/" + uuid.uuid4().hex, Body=b"x")),
"list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)),
"create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])),
"make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)),
"read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)),
"delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)),
"change versioning": attempt("versioning", lambda: c.put_bucket_versioning(
Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})),
}
try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket
result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", []))
except ClientError as e:
result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?")
print(json.dumps(result))
'''
def last_json(output):
return json.loads(output.splitlines()[-1])
def prepare():
alice, bob = new_user("alice"), new_user("bob")
chat = new_chat(alice, bob)
att = upload(alice, chat["id"], images.sample_png()).json
key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'")
return alice, bob, att, key
def test_bucket_is_private():
alice, bob, att, key = prepare()
result = last_json(app_python(ANONYMOUS, key))
for name, reply in result.items():
print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}")
check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}")
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
print("the same file through the API for a chat member -> 200")
print("PASS: the bucket is private: no anonymous read, list, write or policy read")
def test_api_key_is_limited():
alice, bob, att, key = prepare()
result = last_json(app_python(API_KEY_RIGHTS, key))
allowed = {"get own object", "put object", "delete object"} # lab7: листинг — только у janitor
visible = result.pop("visible buckets")
print(f"API key: {'list buckets':<32} -> {visible}")
check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets")
for name, outcome in result.items():
print(f"API key: {name:<32} -> {outcome}")
if name in allowed:
check(outcome == "allowed", f"API key must be allowed to {name}")
else:
check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}")
print("PASS: the API key can only read finished files and put/delete upload sources in staging/; "
"listing, other prefixes, bucket admin, policy and other buckets are denied")
def test_no_open_ports_or_console():
config = json.loads(compose_output("config", "--format", "json"))
s3 = config["services"]["s3"]
check(not s3.get("ports"), "s3 must not publish ports")
check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network")
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, "
"filer UI 8888 are reachable only inside the stand)")
for port in (8333, 18333): # lab7: master/volume/filer слушают только loopback контейнера
sock = socket.socket()
sock.settimeout(1)
refused = sock.connect_ex(("127.0.0.1", port)) != 0
sock.close()
check(refused, f"127.0.0.1:{port} is reachable from the host")
print(f"127.0.0.1:{port} from the host -> connection refused")
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()}
for name, items in sorted(secrets.items()):
if items:
print(f"file secrets of {name}: {items}")
for name in ("api", "janitor", "worker", "dispatcher"):
check(not secrets.get(name), f"{name} must not get file secrets (lab7: credentials come from OpenBao)")
check(secrets.get("s3") == ["s3_config"], "only s3 itself reads the key config")
print("PASS: S3 is reachable only inside the stand (HTTPS); the key config is only in s3; services get their "
"own limited keys from OpenBao, the admin key only s3-init")
def test_bootstrap_is_idempotent():
alice, bob, att, key = prepare()
output = compose_output("run", "--rm", "--no-deps", "s3-init")
print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip()))
check("bucket is private" in output, "bootstrap did not confirm a private bucket")
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
check(got.data == images.sample_png(), "file changed after repeated bootstrap")
upload(alice, att["chat_id"], images.sample_png())
print("after the repeated bootstrap: old file downloads, new upload works")
print("PASS: bucket/user/policy bootstrap can be repeated safely")
if __name__ == "__main__":
run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console,
test_bootstrap_is_idempotent])