160 lines
6.4 KiB
Python
160 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Real storage ACL and dependency failure checks; tokens never leave memory/stdin."""
|
|
|
|
import hashlib
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
|
from client import Client
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def compose(*args):
|
|
return subprocess.run(
|
|
["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True, text=True
|
|
)
|
|
|
|
|
|
def inside(code, payload=None):
|
|
result = subprocess.run(
|
|
["docker", "compose", "exec", "-T", "--index", "1", "api", "python", "-c", code],
|
|
cwd=ROOT,
|
|
input=json.dumps(payload or {}),
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return json.loads(result.stdout)
|
|
|
|
|
|
def request(path, token=None):
|
|
return inside(
|
|
"""import json,sys,time,urllib.request,urllib.error
|
|
p=json.load(sys.stdin);h={"Authorization":"Bearer "+p["token"]} if p.get("token") else {}
|
|
start=time.monotonic()
|
|
try:r=urllib.request.urlopen(urllib.request.Request("http://127.0.0.1:8080"+p["path"],headers=h),timeout=8)
|
|
except urllib.error.HTTPError as e:r=e
|
|
data=r.read()
|
|
print(json.dumps({"status":r.code,"elapsed":time.monotonic()-start,"json":json.loads(data) if r.headers.get_content_type()=="application/json" else None,"sha256":__import__("hashlib").sha256(data).hexdigest()}))
|
|
""",
|
|
{"path": path, "token": token},
|
|
)
|
|
|
|
|
|
def main():
|
|
c = Client("http://127.0.0.1:8084")
|
|
alice, _, _, chat = c.bootstrap()
|
|
attachment, content = c.attachment(alice, chat)
|
|
report = {"checks": [], "faults": []}
|
|
|
|
def check(name, condition):
|
|
if not condition:
|
|
raise AssertionError(name)
|
|
report["checks"].append({"name": name, "passed": True})
|
|
|
|
security = inside(
|
|
"""import json,sys,os,socket,urllib.request,urllib.error,uuid
|
|
from pathlib import Path
|
|
from messenger.storage import ObjectStorage
|
|
from messenger.postgres import PostgresStore
|
|
from botocore.exceptions import ClientError
|
|
p=json.load(sys.stdin);s=ObjectStorage();db=PostgresStore()
|
|
with db.connection() as conn:key=conn.execute("SELECT object_key FROM attachments WHERE id=%s",(p["id"],)).fetchone()["object_key"]
|
|
try:r=urllib.request.urlopen(os.getenv("S3_ENDPOINT")+"/"+s.bucket+"/"+key,timeout=3);anon=r.code;r.close()
|
|
except urllib.error.HTTPError as e:anon=e.code
|
|
try:s.client.create_bucket(Bucket="forbidden-"+uuid.uuid4().hex);denied=False
|
|
except ClientError as e:denied=e.response["ResponseMetadata"]["HTTPStatusCode"]==403
|
|
try:socket.create_connection(("storage",8888),timeout=.5).close();backend=False
|
|
except OSError:backend=True
|
|
try:socket.create_connection(("s3",8888),timeout=.5).close();gateway=False
|
|
except OSError:gateway=True
|
|
print(json.dumps({"anonymous_status":anon,"unrelated_bucket_denied":denied,"backend_unreachable":backend,"gateway_filer_unreachable":gateway,"no_admin_secrets":not Path("/run/secrets/s3_admin_access_key").exists() and not any(k.startswith("S3_ADMIN") for k in os.environ)}));db.close();s.close()
|
|
""",
|
|
{"id": attachment["id"]},
|
|
)
|
|
report["security"] = security
|
|
for key in [
|
|
"unrelated_bucket_denied",
|
|
"backend_unreachable",
|
|
"gateway_filer_unreachable",
|
|
"no_admin_secrets",
|
|
]:
|
|
check(key, security[key])
|
|
check("anonymous_original_denied", security["anonymous_status"] in {401, 403})
|
|
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
|
check("bucket_bootstrap_repeatable", True)
|
|
try:
|
|
compose("stop", "s3")
|
|
ready = request("/health/ready")
|
|
download = request(f"/api/v1/attachments/{attachment['id']}/content", alice["token"])
|
|
check("s3_down_ready_503", ready["status"] == 503 and ready["elapsed"] < 8)
|
|
check("s3_down_content_503", download["status"] == 503 and download["elapsed"] < 8)
|
|
report["faults"].append({"service": "s3", "ready": ready, "download": download})
|
|
finally:
|
|
compose("start", "s3")
|
|
for _ in range(30):
|
|
try:
|
|
downloaded = request(f"/api/v1/attachments/{attachment['id']}/content", alice["token"])
|
|
if downloaded["status"] == 200:
|
|
break
|
|
except (AssertionError, subprocess.CalledProcessError):
|
|
pass
|
|
time.sleep(0.5)
|
|
else:
|
|
raise AssertionError("S3 recovery timed out")
|
|
check(
|
|
"s3_restart_original_unchanged", downloaded["sha256"] == hashlib.sha256(content).hexdigest()
|
|
)
|
|
try:
|
|
compose("pause", "postgres")
|
|
ready = request("/health/ready")
|
|
me = request("/api/v1/users/me", alice["token"])
|
|
check(
|
|
"paused_pg_ready_503_with_client_deadline",
|
|
ready["status"] == 503 and ready["elapsed"] < 8,
|
|
)
|
|
check("paused_pg_auth_503_with_client_deadline", me["status"] == 503 and me["elapsed"] < 8)
|
|
report["faults"].append({"service": "paused_postgres", "ready": ready, "me": me})
|
|
finally:
|
|
compose("unpause", "postgres")
|
|
for _ in range(30):
|
|
try:
|
|
me = request("/api/v1/users/me", alice["token"])
|
|
if me["status"] == 200:
|
|
break
|
|
except subprocess.CalledProcessError:
|
|
pass
|
|
time.sleep(0.5)
|
|
else:
|
|
raise AssertionError("PostgreSQL recovery timed out")
|
|
check("old_session_recovers_after_unpause", me["json"]["id"] == alice["id"])
|
|
inside(
|
|
"from messenger.postgres import PostgresStore; s=PostgresStore(); "
|
|
"c=s.pool.getconn(); c.execute('ALTER TABLE message_requests RENAME TO message_requests_readiness_test'); "
|
|
"s.pool.putconn(c); s.close(); print('{}')"
|
|
)
|
|
try:
|
|
absent_table = request("/health/ready")
|
|
check("missing_required_table_ready_503", absent_table["status"] == 503)
|
|
report["faults"].append({"service": "missing_message_requests", "ready": absent_table})
|
|
finally:
|
|
inside(
|
|
"from messenger.postgres import PostgresStore; s=PostgresStore(); "
|
|
"c=s.pool.getconn(); c.execute('ALTER TABLE message_requests_readiness_test RENAME TO message_requests'); "
|
|
"s.pool.putconn(c); s.close(); print('{}')"
|
|
)
|
|
check("schema_restore_ready_200", request("/health/ready")["status"] == 200)
|
|
Path(ROOT / "evidence/storage-checks.json").write_text(
|
|
json.dumps(report, ensure_ascii=False, indent=2) + "\n"
|
|
)
|
|
print(json.dumps({"passed": len(report["checks"])}))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|