forked from mtusi2026/mtusi2026lab5
68 lines
2.9 KiB
Python
68 lines
2.9 KiB
Python
#!/usr/bin/env python3
|
||
"""Reaper восстанавливает job после паузы worker’а дольше lease (60с)."""
|
||
import sys, time
|
||
from pathlib import Path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||
from tests.client import Client, check
|
||
from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes
|
||
|
||
def main():
|
||
c = Client()
|
||
a, b, e, ch = c.bootstrap()
|
||
content = png_bytes(4000, 4000)
|
||
|
||
# 1) Готовим job, но worker не даём её взять.
|
||
docker('pause', 'worker')
|
||
try:
|
||
att = upload(c, a, ch, content)
|
||
finally:
|
||
pass # unpause ниже — после того, как настроим SIGSTOP
|
||
|
||
# 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP.
|
||
# Используем docker kill --signal=SIGSTOP, чтобы остановить процесс
|
||
# (не контейнер). Контейнер "жив", но worker заморожен.
|
||
docker('unpause', 'worker')
|
||
|
||
# 3) Ждём processing; как только взял — сразу SIGSTOP.
|
||
deadline = time.monotonic() + 15
|
||
job = None
|
||
while time.monotonic() < deadline:
|
||
st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||
if st == 'processing':
|
||
# Останавливаем процесс, пока он внутри processing.
|
||
docker('kill', '-s', 'SIGSTOP', 'worker')
|
||
job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id'])
|
||
break
|
||
if st == 'done':
|
||
fail('job finished too fast to simulate stuck worker')
|
||
time.sleep(0.02)
|
||
|
||
check(job is not None, 'worker never entered processing')
|
||
|
||
# 4) Фиксируем lease_expires_at на момент SIGSTOP.
|
||
lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job)
|
||
ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease))
|
||
|
||
# 5) Ждём, пока lease протухнет и reaper вернёт job в queued.
|
||
# lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом.
|
||
deadline = time.monotonic() + 120
|
||
requeued = False
|
||
while time.monotonic() < deadline:
|
||
st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job)
|
||
if st == 'queued':
|
||
requeued = True
|
||
break
|
||
time.sleep(2)
|
||
check(requeued, 'reaper did not requeue stuck job (status=%s)' % st)
|
||
ok('reaper requeued the stuck job')
|
||
|
||
# 6) Возобновляем worker: он должен доделать job до ready.
|
||
docker('kill', '-s', 'SIGCONT', 'worker')
|
||
att = wait_status(c, a, att['id'], 'ready', seconds=120)
|
||
ok('ready after unstuck')
|
||
|
||
if __name__ == '__main__':
|
||
try:
|
||
main()
|
||
except Exception as exc:
|
||
fail(str(exc)) |