#!/usr/bin/env python3 """Reaper восстанавливает job после паузы worker’а дольше lease (60с).""" import sys, time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from tests.client import Client, check from student_tests.common import upload, wait_status, psql, docker, ok, fail, png_bytes def main(): c = Client() a, b, e, ch = c.bootstrap() content = png_bytes(4000, 4000) # 1) Готовим job, но worker не даём её взять. docker('pause', 'worker') try: att = upload(c, a, ch, content) finally: pass # unpause ниже — после того, как настроим SIGSTOP # 2) Возобновляем worker: он возьмёт job и сразу поймает SIGSTOP. # Используем docker kill --signal=SIGSTOP, чтобы остановить процесс # (не контейнер). Контейнер "жив", но worker заморожен. docker('unpause', 'worker') # 3) Ждём processing; как только взял — сразу SIGSTOP. deadline = time.monotonic() + 15 job = None while time.monotonic() < deadline: st = psql("SELECT status FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) if st == 'processing': # Останавливаем процесс, пока он внутри processing. docker('kill', '-s', 'SIGSTOP', 'worker') job = psql("SELECT job_id FROM attachment_jobs WHERE attachment_id='%s'" % att['id']) break if st == 'done': fail('job finished too fast to simulate stuck worker') time.sleep(0.02) check(job is not None, 'worker never entered processing') # 4) Фиксируем lease_expires_at на момент SIGSTOP. lease = psql("SELECT lease_expires_at FROM attachment_jobs WHERE job_id='%s'" % job) ok('worker stopped at processing (job=%s, lease_expires_at=%s)' % (job, lease)) # 5) Ждём, пока lease протухнет и reaper вернёт job в queued. # lease=60s, reaper=30s ⇒ до 100s; берём 120 с запасом. deadline = time.monotonic() + 120 requeued = False while time.monotonic() < deadline: st = psql("SELECT status FROM attachment_jobs WHERE job_id='%s'" % job) if st == 'queued': requeued = True break time.sleep(2) check(requeued, 'reaper did not requeue stuck job (status=%s)' % st) ok('reaper requeued the stuck job') # 6) Возобновляем worker: он должен доделать job до ready. docker('kill', '-s', 'SIGCONT', 'worker') att = wait_status(c, a, att['id'], 'ready', seconds=120) ok('ready after unstuck') if __name__ == '__main__': try: main() except Exception as exc: fail(str(exc))