78 lines
4.3 KiB
Python
78 lines
4.3 KiB
Python
"""lab4–lab5, уровень 3: недоступность S3.
|
||
|
||
make failure-s3 (= python student-tests/failure_s3.py)
|
||
|
||
С lab5 S3 входит в /health/ready: без него нельзя принять файл в staging, а это обязательный путь
|
||
(без staging нельзя честно ответить 202). Поэтому при остановке контейнера s3 (SeaweedFS):
|
||
- каждая реплика: /health/live = 200 (процесс жив), /health/ready = 503 (не может обслужить путь);
|
||
- HAProxy выводит обе реплики из балансировки и сам отвечает 503 в формате ошибки контракта —
|
||
быстро, без зависаний и без 500;
|
||
после запуска S3 реплики возвращаются, файл, загруженный до сбоя, скачивается без изменений,
|
||
новая загрузка снова 202 -> ready.
|
||
"""
|
||
import hashlib
|
||
import time
|
||
|
||
from filegen import SAMPLE_PNG, upload_when_ready, wait_status, wait_until
|
||
from helpers import Client, Failure, check, run
|
||
from stand import api_containers, compose_quiet, lb_summary, lb_up_count, ps_table, replica_status, try_call
|
||
|
||
|
||
def main():
|
||
api = Client()
|
||
replicas = api_containers()
|
||
alice, bob, _eve, chat = api.bootstrap()
|
||
att = upload_when_ready(api, alice, chat, name="before.png")["json"]
|
||
wait_status(api, bob, att["id"])
|
||
content = f"/api/v1/attachments/{att['id']}/content"
|
||
compose_quiet("stop", "s3")
|
||
stopped = time.monotonic()
|
||
try:
|
||
print("=== s3 stopped")
|
||
print(ps_table())
|
||
wait_until(lambda: lb_up_count() == 0, 30, "HAProxy kept a replica that cannot reach S3")
|
||
took = time.monotonic() - stopped
|
||
print(f"both api replicas are out of the balancer after {took:.1f} s: {lb_summary()}")
|
||
for container in replicas:
|
||
live, ready = replica_status(container, "/health/live"), replica_status(container, "/health/ready")
|
||
print(f"replica {container[:12]}: /health/live {live}, /health/ready {ready}")
|
||
check(live == 200 and ready == 503, "expected live 200 and ready 503")
|
||
for method, path, body, ctype in (("GET", "/health/ready", None, None),
|
||
("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=x.png",
|
||
SAMPLE_PNG, "image/png"),
|
||
("GET", content, None, None),
|
||
("GET", f"/api/v1/chats/{chat['id']}/messages", None, None)):
|
||
started = time.monotonic()
|
||
reply = try_call(method, path, raw=body, token=alice["token"],
|
||
headers={"Content-Type": ctype} if ctype else None, timeout=15)
|
||
check(reply is not None and reply.status == 503, f"{method} {path}: expected 503")
|
||
check(reply.json["error"]["code"] == "unavailable" and reply.headers.get("x-request-id"),
|
||
"the balancer must answer in the contract error format")
|
||
print(f"{method} {path.split('?')[0]} -> {reply.status} {reply.json['error']['code']} "
|
||
f"in {time.monotonic() - started:.2f} s (answered by the balancer)")
|
||
finally:
|
||
compose_quiet("start", "s3")
|
||
print("=== s3 started")
|
||
started = time.monotonic()
|
||
wait_until(lambda: lb_up_count() == len(replicas), 120, "replicas did not come back")
|
||
print(f"replicas are back in the balancer after {time.monotonic() - started:.1f} s: {lb_summary()}")
|
||
|
||
def downloaded():
|
||
try:
|
||
return api.call("GET", content, actor=bob)["bytes"] == SAMPLE_PNG
|
||
except Failure:
|
||
return False
|
||
|
||
wait_until(downloaded, 60, "the old file is not downloadable after S3 restart")
|
||
print(f"the file uploaded before the outage is intact (sha256 {hashlib.sha256(SAMPLE_PNG).hexdigest()[:16]}...)")
|
||
new = upload_when_ready(api, alice, chat, name="after.png")["json"]
|
||
wait_status(api, bob, new["id"])
|
||
print(f"new upload -> 202 -> ready ({new['id']})")
|
||
print(ps_table())
|
||
print("PASS: S3 down -> replicas not ready (live 200, ready 503), the balancer answers 503 in the contract "
|
||
"format, no 500 and no hanging; after S3 is back the original is intact and uploads work again")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main)
|