Files
mtusi2026lab5-fork/student_tests/lease.py
2026-11-16 20:30:12 +03:00

68 lines
2.9 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Reaper восстанавливает job после паузы worker’а дольше lease (60с)."""
import sys, time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from tests.client import Client, check
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
def main():
c = Client()
a, b, e, ch = c.bootstrap()
content = png_bytes(4000, 4000)
# 1) Готовим job, но worker не даём её взять.
docker('pause', 'worker')
try:
att = upload(c, a, ch, content)
finally:
pass # unpause ниже — после того, как настроим SIGSTOP
# 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP.
# Используем docker kill --signal=SIGSTOP, чтобы остановить процесс
# (не контейнер). Контейнер "жив", но worker заморожен.
docker('unpause', 'worker')
# 3) Ждём processing; как только взял — сразу SIGSTOP.
deadline = time.monotonic() + 15
job = None
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
if st == 'processing':
# Останавливаем процесс, пока он внутри processing.
docker('kill', '-s', 'SIGSTOP', 'worker')
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
break
if st == 'done':
fail('job finished too fast to simulate stuck worker')
time.sleep(0.02)
check(job is not None, 'worker never entered processing')
# 4) Фиксируем lease_expires_at на момент SIGSTOP.
lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job)
ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease))
# 5) Ждём, пока lease протухнет и reaper вернёт job в queued.
# lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом.
deadline = time.monotonic() + 120
requeued = False
while time.monotonic() < deadline:
st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job)
if st == 'queued':
requeued = True
break
time.sleep(2)
check(requeued, 'reaper did not requeue stuck job (status=%s)' % st)
ok('reaper requeued the stuck job')
# 6) Возобновляем worker: он должен доделать job до ready.
docker('kill', '-s', 'SIGCONT', 'worker')
att = wait_status(c, a, att['id'], 'ready', seconds=120)
ok('ready after unstuck')
if __name__ == '__main__':
try:
main()
except Exception as exc:
fail(str(exc))