Files
mtusi2026lab7/student-tests/lifecycle.py
T
2026-11-11 13:53:51 +03:00

202 lines
8.2 KiB
Python

"""Prove real credential rejection, service token renewal and the sealed lease limit."""
import argparse
import json
from pathlib import Path
import ssl
import subprocess
import sys
import time
import urllib.request
from urllib.error import HTTPError
ROOT = Path(__file__).resolve().parents[1]
sys.path[:0] = [str(ROOT), str(ROOT / "tests")]
from scripts.bao_operator import ADDRESS, call, login, rotate_db, rotate_s3, unseal
from scripts.wait_ready import wait
from src.bao import client, request
from client import Client, check
CA = "secrets/authority/ca.crt"
COMPOSE = ["docker", "compose"]
def code(args):
subprocess.run(args, check=True, stdin=subprocess.DEVNULL)
def image_id(service):
result = subprocess.check_output(COMPOSE + ["ps", "-q", service], text=True).splitlines()
return sorted({subprocess.check_output(["docker", "inspect", "-f", "{{.Image}}", cid],
text=True).strip() for cid in result})
def verify_db(dsn, expected):
result = subprocess.run(
COMPOSE + ["exec", "-T", "--index", "1", "api", "python", "-c", """
import psycopg,sys
try:
with psycopg.connect(sys.stdin.read(),connect_timeout=3) as db:
assert db.execute('SELECT current_user').fetchone()[0]=='lab_api'
print('connected')
except psycopg.OperationalError as e:
print('invalid_password' if 'password authentication failed' in str(e) else 'other_error')
"""], input=dsn, text=True, capture_output=True, check=True)
check(result.stdout.strip() == expected, f"DB connection expected {expected}")
def db_rotation():
token = login()["client_token"]
old = call("messenger/data/api", token=token)["data"]["data"]["DATABASE_URL"]
verify_db(old, "connected")
previous = image_id("api")
started = time.monotonic()
rotate_db()
verify_db(old, "invalid_password")
print("PASS old API DB password rejected on fresh TLS connection")
code(COMPOSE + ["up", "-d", "--force-recreate", "--no-deps", "api"])
wait(2, 60)
new = call("messenger/data/api", token=token)["data"]["data"]["DATABASE_URL"]
verify_db(new, "connected")
check(previous == image_id("api"), "API image unexpectedly changed")
print("PASS new API DB password works with same image",
json.dumps({"recovery_seconds": round(time.monotonic() - started, 3)}))
def leaf_rotation():
previous = image_id("lb")
started = time.monotonic()
code(["python3", "scripts/local_pki.py", "rotate-lb"])
code(COMPOSE + ["restart", "lb"])
wait(2, 60)
code(COMPOSE + ["run", "--rm", "--no-deps", "-T", "audit",
"python", "/app/student-tests/trust.py", "rotation"])
check(previous == image_id("lb"), "LB image unexpectedly changed")
print("PASS leaf revoked on both APIs without rebuild",
json.dumps({"recovery_seconds": round(time.monotonic() - started, 3)}))
def s3_status(values, expected):
result = subprocess.run(
COMPOSE + ["exec", "-T", "--index", "1", "api", "python", "-c", """
import boto3,json,sys
from botocore.exceptions import ClientError
v=json.load(sys.stdin)
s3=boto3.client("s3",endpoint_url="https://s3:9000",region_name="us-east-1",verify="/pki/ca.crt",
aws_access_key_id=v["S3_ACCESS_KEY_ID"],aws_secret_access_key=v["S3_SECRET_ACCESS_KEY"])
try:
s3.list_objects_v2(Bucket="messenger-attachments",Prefix="originals/",MaxKeys=1)
print("accepted")
except ClientError as exc:
print("rejected" if exc.response["ResponseMetadata"]["HTTPStatusCode"]==403 else "other_error")
"""], input=json.dumps(values), text=True, capture_output=True, check=True)
check(result.stdout.strip() == expected, f"S3 credentials expected {expected}")
def s3_rotation():
token = login()["client_token"]
path = "messenger/data/api"
old = call(path, token=token)["data"]["data"]
s3_status(old, "accepted")
previous = image_id("api")
started = time.monotonic()
rotate_s3()
s3_status(old, "rejected")
print("PASS old S3 secret rejected on fresh TLS connection")
code(COMPOSE + ["up", "-d", "--force-recreate", "--no-deps", "api"])
wait(2, 60)
new = call(path, token=token)["data"]["data"]
s3_status(new, "accepted")
check(previous == image_id("api"), "API image unexpectedly changed")
print("PASS new scoped S3 secret accepted with same image",
json.dumps({"recovery_seconds": round(time.monotonic() - started, 3)}))
def collector_rotation():
probe = COMPOSE + ["run", "--rm", "--no-deps", "-T", "audit", "python", "/app/student-tests/trust.py"]
code(probe + ["collector-current"])
images = [image_id(role) for role in ("otel", "loki", "tempo")]
started = time.monotonic()
code(["python3", "scripts/local_pki.py", "rotate-collector"])
code(COMPOSE + ["stop", "otel", "logs", "traces"])
code(COMPOSE + ["restart", "loki", "tempo"])
code(COMPOSE + ["start", "logs", "traces", "otel"])
end = time.monotonic() + 30
while True:
try:
code(probe + ["collector-rotation"])
break
except subprocess.CalledProcessError:
if time.monotonic() >= end:
raise
time.sleep(2)
check(images == [image_id(role) for role in ("otel", "loki", "tempo")], "image unexpectedly changed")
print("PASS Collector exporter leaf rotated without rebuild",
json.dumps({"recovery_seconds": round(time.monotonic() - started, 3)}))
def ingress_status(path):
ctx = ssl.create_default_context(cafile=CA)
try:
with urllib.request.urlopen("https://localhost:8443" + path, context=ctx, timeout=4) as res:
return res.status
except HTTPError as exc:
return exc.code
def bao_lifecycle():
pki = Path("secrets/runtime/api")
opener = client(*(str(pki / f) for f in ("service.crt", "service.key", "ca.crt")))
def api(path, data=None, token=None):
return request(opener, ADDRESS, path, data, token)
auth = api("auth/cert/login", {"name": "api"})["auth"]
original_ttl = auth["lease_duration"]
token = auth["client_token"]
renewed = api("auth/token/renew-self", {"increment": f"{original_ttl}s"}, token)["auth"]
check(renewed["renewable"] and renewed["lease_duration"] > 0, "token renewal")
api("messenger/data/api", token=token)
print(f"PASS role token renewal, lease {original_ttl}s → {renewed['lease_duration']}s")
operator = login()["client_token"]
call("auth/token/revoke-accessor", {"accessor": auth["accessor"]}, operator)
try:
api("messenger/data/api", token=token)
except HTTPError as exc:
check(exc.code == 403, "revocation should deny with HTTP 403")
else:
raise AssertionError("revoked token still works")
print("PASS revoked scoped token denied on fresh HTTPS request")
c = Client(ca_file=CA)
actor = c.user("seal")
check(ingress_status("/health/ready") == 200, "pre-seal readiness")
started = time.monotonic()
call("sys/seal", token=operator, method="PUT")
try:
immediate = ingress_status("/health/ready")
check(immediate == 200, "expected short grace before expiry")
deadline = time.monotonic() + original_ttl + 12
consecutive = 0
while time.monotonic() < deadline and consecutive < 4:
consecutive = consecutive + 1 if ingress_status("/health/ready") == 503 else 0
time.sleep(.4)
expiry_seconds = time.monotonic() - started
check(consecutive == 4, "sealed credential grace exceeded")
check(ingress_status("/health/live") == 200, "API process died while sealed")
print("PASS sealed: readiness 200 → 503, liveness 200",
json.dumps({"after_seconds": round(expiry_seconds, 3)}))
finally:
resumed = time.monotonic()
unseal()
wait(2, 65)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
print("PASS after unseal same user session resumes",
json.dumps({"recovery_seconds": round(time.monotonic() - resumed, 3)}))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("scenario", choices=("db", "s3", "leaf", "collector", "bao"))
{"db": db_rotation, "s3": s3_rotation, "leaf": leaf_rotation,
"collector": collector_rotation, "bao": bao_lifecycle}[parser.parse_args().scenario]()