forked from mtusi2026/mtusi2026lab4
89 lines
2.7 KiB
Python
89 lines
2.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Verify uploads return an explicit 503 while the Compose S3 service is stopped."""
|
|
import json
|
|
from pathlib import Path
|
|
import subprocess
|
|
import time
|
|
|
|
from client import Client, ROOT, arguments, check, run
|
|
|
|
|
|
def compose(*args):
|
|
subprocess.run(
|
|
["docker", "compose", *args],
|
|
cwd=ROOT,
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
|
|
|
|
def healthy_api_container():
|
|
result = subprocess.run(
|
|
["docker", "compose", "ps", "-q", "api"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
for container in result.stdout.split():
|
|
health = subprocess.run(
|
|
["docker", "inspect", "--format", "{{.State.Health.Status}}", container],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if health.stdout.strip() == "healthy":
|
|
return container
|
|
raise AssertionError("no healthy API replica is available")
|
|
|
|
|
|
def main():
|
|
args = arguments("S3 outage behavior; stops and restores the local Compose S3 service")
|
|
client = Client(args.base_url, args.ca_file, timeout=15)
|
|
alice, _, _, chat = client.bootstrap()
|
|
fixture = (Path(__file__).parent / "fixtures" / "sample.png").read_bytes()
|
|
api_container = healthy_api_container()
|
|
stopped = False
|
|
try:
|
|
compose("stop", "s3")
|
|
stopped = True
|
|
payload = json.dumps(
|
|
{"alice": alice, "chat": chat, "fixture": fixture.hex()}
|
|
)
|
|
worker = subprocess.run(
|
|
[
|
|
"docker",
|
|
"exec",
|
|
"-i",
|
|
api_container,
|
|
"python",
|
|
"tests/s3_failure_worker.py",
|
|
],
|
|
cwd=ROOT,
|
|
input=payload,
|
|
text=True,
|
|
capture_output=True,
|
|
check=False,
|
|
)
|
|
check(worker.returncode == 0, "S3 failure worker failed: " + worker.stderr.strip())
|
|
check(worker.stdout.strip() == "PASS", "API did not return 503 for unavailable S3")
|
|
finally:
|
|
if stopped:
|
|
compose("up", "-d", "s3")
|
|
compose("run", "--rm", "--no-deps", "s3-bootstrap")
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
client.call("GET", "/health/ready", schema="Health")
|
|
break
|
|
except AssertionError:
|
|
time.sleep(1)
|
|
else:
|
|
raise AssertionError("API did not become ready after S3 recovery")
|
|
print("PASS: upload returned 503 during S3 outage; bucket and API recovered with volumes intact.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|