forked from mtusi2026/mtusi2026lab5
92 lines
5.1 KiB
Python
92 lines
5.1 KiB
Python
"""lab5, уровень 4: worker падает после записи объектов, но до ack — повтор даёт тот же результат.
|
|
|
|
make worker-crash (= python student-tests/worker_crash.py)
|
|
|
|
A. worker_after_objects: оригинал и три варианта уже записаны в S3, COMMIT результата и ack — нет.
|
|
Процесс worker падает (os._exit, как SIGKILL), Docker перезапускает контейнер. Брокер снова
|
|
доставляет неподтверждённое сообщение; новый процесс видит, что lease держит прежний запуск
|
|
ЭТОГО ЖЕ контейнера (значит, тот мёртв), считает попытку неудачной и назначает следующую.
|
|
Вторая попытка пишет варианты в ТЕ ЖЕ ключи (ключ эффекта attachment_id/версия/вариант) теми же
|
|
байтами, коммитит и подтверждает. Итог: ready, попытка 2, ровно 3 строки и 3 объекта вариантов,
|
|
SHA-256 вариантов равны тем, что записала упавшая попытка (вычисляются по самим объектам в S3).
|
|
B. worker_after_commit: результат закоммичен, ack не отправлен, worker падает. Повторная доставка
|
|
видит ready и подтверждает сообщение без обработки (duplicate_ignored); попытка остаётся 1.
|
|
"""
|
|
import json
|
|
import time
|
|
|
|
import ops
|
|
from filegen import upload_when_ready, wait_status
|
|
from helpers import Client, check, run
|
|
|
|
SNAPSHOT = """
|
|
import hashlib, json
|
|
from messenger import objects, config
|
|
out = {}
|
|
for key, _ in objects.list_objects("variants/%s/"):
|
|
body = objects.client().get_object(Bucket=config.S3_BUCKET, Key=key)["Body"].read()
|
|
out[key] = hashlib.sha256(body).hexdigest()
|
|
print(json.dumps(out))
|
|
"""
|
|
|
|
|
|
def snapshot(attachment_id):
|
|
return json.loads(ops.python("api", SNAPSHOT % attachment_id).splitlines()[-1])
|
|
|
|
|
|
def wait_until(condition, seconds, message):
|
|
deadline = time.monotonic() + seconds
|
|
while True:
|
|
value = condition()
|
|
if value:
|
|
return value
|
|
check(time.monotonic() < deadline, message)
|
|
time.sleep(0.5)
|
|
|
|
|
|
def main():
|
|
api = Client(timeout=15)
|
|
worker = ops.wait_healthy("worker", 1)[0]
|
|
alice, bob, _eve, chat = api.bootstrap()
|
|
|
|
# ---------- A ----------
|
|
restarts = ops.restarts(worker)
|
|
ops.arm("worker", "worker_after_objects", container=worker)
|
|
att = upload_when_ready(api, alice, chat, name="worker-crash-a.png")["json"]
|
|
print(f"A. worker_after_objects armed; upload -> 202 ({att['id']})")
|
|
wait_until(lambda: ops.restarts(worker) > restarts, 60, "the worker did not crash")
|
|
crashed = snapshot(att["id"])
|
|
print(f" the worker crashed after writing objects (restart count {restarts} -> {ops.restarts(worker)}); "
|
|
f"objects left by the crashed attempt: {len(crashed)}")
|
|
ready = wait_status(api, bob, att["id"], seconds=120)
|
|
job = ops.job(att["id"])
|
|
final = snapshot(att["id"])
|
|
check(job["job"]["attempt"] == 2, "the retry must be the second attempt")
|
|
check(len(job["variants"]) == 3 and len(final) == 3, "exactly one set of variants expected")
|
|
check(all(final[key] == digest for key, digest in crashed.items()), "the retry produced different bytes")
|
|
check({v["sha256"] for v in ready["variants"]} == set(final.values()), "metadata does not match the objects")
|
|
print(" events: " + " -> ".join(f"{e['event']}({e['attempt']})" for e in job["events"]))
|
|
print(f" ready at attempt 2; same keys and the same SHA-256 as the crashed attempt wrote: "
|
|
f"{sorted(k.rsplit('/', 1)[1] for k in final)}; 3 variant rows")
|
|
ops.wait_healthy("worker", 1)
|
|
|
|
# ---------- B ----------
|
|
restarts = ops.restarts(worker)
|
|
ops.arm("worker", "worker_after_commit", container=worker)
|
|
att = upload_when_ready(api, alice, chat, name="worker-crash-b.png")["json"]
|
|
print(f"B. worker_after_commit armed; upload -> 202 ({att['id']})")
|
|
wait_until(lambda: ops.restarts(worker) > restarts, 60, "the worker did not crash")
|
|
wait_status(api, bob, att["id"], seconds=60)
|
|
job = wait_until(lambda: (lambda j: j if any(e["event"] == "duplicate_ignored" for e in j["events"]) else None)(
|
|
ops.job(att["id"])), 90, "the redelivered message was not seen")
|
|
check(job["job"]["attempt"] == 1 and len(job["variants"]) == 3, "the duplicate must not reprocess the job")
|
|
print(" events: " + " -> ".join(f"{e['event']}({e['attempt']})" for e in job["events"]))
|
|
print(" the redelivered message found the job ready and was acknowledged without processing")
|
|
ops.wait_healthy("worker", 1)
|
|
print("PASS: a worker crash after writing objects but before ack is finished by a retry with the same result; "
|
|
"a crash after COMMIT but before ack ends with a no-op redelivery")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main)
|