Files
mtusi2026lab5/student-tests/failure_s3.py
T
2026-10-25 21:43:03 +03:00

78 lines
4.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""lab4–lab5, уровень 3: недоступность S3.
make failure-s3 (= python student-tests/failure_s3.py)
С lab5 S3 входит в /health/ready: без него нельзя принять файл в staging, а это обязательный путь
(без staging нельзя честно ответить 202). Поэтому при остановке контейнера s3 (SeaweedFS):
- каждая реплика: /health/live = 200 (процесс жив), /health/ready = 503 (не может обслужить путь);
- HAProxy выводит обе реплики из балансировки и сам отвечает 503 в формате ошибки контракта —
быстро, без зависаний и без 500;
после запуска S3 реплики возвращаются, файл, загруженный до сбоя, скачивается без изменений,
новая загрузка снова 202 -> ready.
"""
import hashlib
import time
from filegen import SAMPLE_PNG, upload_when_ready, wait_status, wait_until
from helpers import Client, Failure, check, run
from stand import api_containers, compose_quiet, lb_summary, lb_up_count, ps_table, replica_status, try_call
def main():
api = Client()
replicas = api_containers()
alice, bob, _eve, chat = api.bootstrap()
att = upload_when_ready(api, alice, chat, name="before.png")["json"]
wait_status(api, bob, att["id"])
content = f"/api/v1/attachments/{att['id']}/content"
compose_quiet("stop", "s3")
stopped = time.monotonic()
try:
print("=== s3 stopped")
print(ps_table())
wait_until(lambda: lb_up_count() == 0, 30, "HAProxy kept a replica that cannot reach S3")
took = time.monotonic() - stopped
print(f"both api replicas are out of the balancer after {took:.1f} s: {lb_summary()}")
for container in replicas:
live, ready = replica_status(container, "/health/live"), replica_status(container, "/health/ready")
print(f"replica {container[:12]}: /health/live {live}, /health/ready {ready}")
check(live == 200 and ready == 503, "expected live 200 and ready 503")
for method, path, body, ctype in (("GET", "/health/ready", None, None),
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=x.png",
SAMPLE_PNG, "image/png"),
("GET", content, None, None),
("GET", f"/api/v1/chats/{chat['id']}/messages", None, None)):
started = time.monotonic()
reply = try_call(method, path, raw=body, token=alice["token"],
headers={"Content-Type": ctype} if ctype else None, timeout=15)
check(reply is not None and reply.status == 503, f"{method} {path}: expected 503")
check(reply.json["error"]["code"] == "unavailable" and reply.headers.get("x-request-id"),
"the balancer must answer in the contract error format")
print(f"{method} {path.split('?')[0]} -> {reply.status} {reply.json['error']['code']} "
f"in {time.monotonic() - started:.2f} s (answered by the balancer)")
finally:
compose_quiet("start", "s3")
print("=== s3 started")
started = time.monotonic()
wait_until(lambda: lb_up_count() == len(replicas), 120, "replicas did not come back")
print(f"replicas are back in the balancer after {time.monotonic() - started:.1f} s: {lb_summary()}")
def downloaded():
try:
return api.call("GET", content, actor=bob)["bytes"] == SAMPLE_PNG
except Failure:
return False
wait_until(downloaded, 60, "the old file is not downloadable after S3 restart")
print(f"the file uploaded before the outage is intact (sha256 {hashlib.sha256(SAMPLE_PNG).hexdigest()[:16]}...)")
new = upload_when_ready(api, alice, chat, name="after.png")["json"]
wait_status(api, bob, new["id"])
print(f"new upload -> 202 -> ready ({new['id']})")
print(ps_table())
print("PASS: S3 down -> replicas not ready (live 200, ready 503), the balancer answers 503 in the contract "
"format, no 500 and no hanging; after S3 is back the original is intact and uploads work again")
if __name__ == "__main__":
run(main)