54 lines
3.0 KiB
Python
54 lines
3.0 KiB
Python
"""lab4–lab5, уровень 3: upload -> СРАЗУ остановка обслужившей реплики -> ready и download через другую.
|
||
|
||
make attachment-failover (= python student-tests/attachment_failover.py)
|
||
|
||
С lab5 реплика останавливается сразу после ответа 202, ещё до обработки. Это доказывает две вещи:
|
||
- остановка API после ответа не теряет работу: оригинал уже в staging S3, задание — в PostgreSQL,
|
||
worker доводит вложение до ready без этой реплики;
|
||
- оригинал не лежит на диске реплики: другая реплика отдаёт тот же файл байт в байт (SHA-256).
|
||
В конце остановленная реплика запускается обратно (`docker start`).
|
||
"""
|
||
import hashlib
|
||
|
||
from filegen import upload_when_ready, wait_status, wait_until
|
||
from helpers import Client, check, run
|
||
from stand import docker, lb_summary, lb_up_count, ps_table, wait_balanced
|
||
|
||
|
||
def main():
|
||
api = Client()
|
||
replicas = wait_balanced(2)
|
||
alice, bob, _eve, chat = api.bootstrap()
|
||
reply = upload_when_ready(api, alice, chat, name="failover.png")
|
||
attachment, served = reply["json"], reply["headers"]["X-Instance-Id"]
|
||
target = [c for c in replicas if c.startswith(served)]
|
||
check(len(target) == 1, f"X-Instance-Id {served} does not match a running api container")
|
||
print(f"uploaded {attachment['size_bytes']} bytes -> 202 {attachment['status']}, "
|
||
f"sha256 {attachment['sha256'][:16]}..., served by api replica {served}")
|
||
print(ps_table())
|
||
docker("stop", target[0])
|
||
try:
|
||
wait_until(lambda: lb_up_count() == len(replicas) - 1, 30, "HAProxy did not remove the stopped replica")
|
||
print(f"stopped {served} (docker stop) right after 202; HAProxy: {lb_summary()}")
|
||
ready = wait_status(api, bob, attachment["id"], seconds=90)
|
||
print(f"the worker finished the job without that replica: status {ready['status']}, "
|
||
f"variants {[v['name'] for v in ready['variants']]}")
|
||
seen = set()
|
||
for _ in range(4):
|
||
got = api.call("GET", f"/api/v1/attachments/{attachment['id']}/content", actor=bob)
|
||
check(hashlib.sha256(got["bytes"]).hexdigest() == attachment["sha256"], "downloaded file differs")
|
||
seen.add(got["headers"]["X-Instance-Id"])
|
||
check(served not in seen, "the stopped replica still answered")
|
||
print(f"downloaded 4 times through {sorted(seen)}: same SHA-256 as uploaded")
|
||
finally:
|
||
docker("start", target[0])
|
||
wait_balanced(len(replicas))
|
||
print(f"restored {served}; HAProxy: {lb_summary()}")
|
||
print(ps_table())
|
||
print("PASS: the replica that accepted the upload was stopped right after 202; the job still became ready and "
|
||
"another replica returned the same bytes (the original lives in S3, not on the replica)")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run(main)
|