fix: продление аренды задач и контроль зависшей обработки

- Добавлен heartbeat для продления lease долгих задач.
- Добавлен общий таймаут с остановкой дерева процессов.
- Сохранена защита от устаревших исполнителей.
- Добавлены тесты и обновлена документация.
This commit is contained in:
2026-11-20 14:00:53 +03:00
parent eca7c0f763
commit fd501cbf68
12 changed files with 441 additions and 12 deletions
+38 -2
View File
@@ -27,12 +27,14 @@ def main():
base = 'http://127.0.0.1:' + str(port)
env = dict(os.environ, COMPOSE_PROJECT_NAME=project, HOST_PORT=str(port), BASE_URL=base,
API_REPLICAS='2', ENABLE_FAULT_INJECTION='1', JOB_TIMEOUT_SECONDS='5',
JOB_LEASE_SECONDS='15', JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15')
JOB_LEASE_SECONDS='15', JOB_TASK_TIMEOUT_SECONDS='25',
JOB_MAX_ATTEMPTS='5', JOB_BACKOFF_SECONDS='0.15')
result = {'utc': datetime.now(timezone.utc).isoformat(), 'command': './scripts/check-docker.sh',
'project': project, 'host': platform.system() + ' ' + platform.machine(),
'source_commit': subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip(),
'working_tree': True, 'configuration': {'concurrency': 2, 'prefetch': 1, 'attempts': 5,
'timeout_seconds': 5, 'lease_seconds': 15, 'backoff_base_seconds': 0.15}, 'checks': {}}
'timeout_seconds': 5, 'task_timeout_seconds': 25, 'lease_seconds': 15,
'backoff_base_seconds': 0.15}, 'checks': {}}
def command(args, timeout=240, check=True):
return subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, check=check)
@@ -180,6 +182,40 @@ db.pool.close()
assert variants['thumbnail']['size'] == [256, 160] and variants['crop']['size'] == [128, 128]
record('worker_offline_api_broker_restart', attachment_id=att['id'], job_id=jid, message_id=msg['id'], variants=variants)
# Исправная попытка живёт дольше исходного lease без ложного retry.
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'before_download', 'delay:20')
compose('start', 'publisher')
wait(lambda: hit(jid, 'before_download'), description='long task started')
initial_lease = sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'")
assert worker_code('''from uuid import uuid4
from src import db,jobs
db.pool.open(wait=True)
with db.connection() as c:
j=jobs.locked(c,''' + repr(jid) + ''')
j['lease_token']=uuid4()
print(jobs.renew(j))
db.pool.close()
''') == 'False'
wait(lambda: sql("SELECT lease_until FROM attachment_jobs WHERE id='" + jid + "'") != initial_lease,
seconds=12, description='heartbeat extended lease')
assert_variants(att, png)
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '1'
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND event='retry'") == '0'
record('long_task_heartbeat', job_id=jid, delay_seconds=20, lease_seconds=15, attempt=1)
# Heartbeat не сохраняет зависшую работу бесконечно: общий deadline.
compose('stop', '--timeout', '3', 'publisher')
att = upload(); jid = job_id(att)
fault(jid, 'before_download', 'timeout')
compose('start', 'publisher')
assert_variants(att, png)
assert hit(jid, 'before_download')
assert sql("SELECT count(*) FROM attachment_job_events WHERE job_id='" + jid + "' AND error_code='processing_timeout'") == '1'
assert sql("SELECT attempt FROM attachment_jobs WHERE id='" + jid + "'") == '2'
record('whole_task_timeout', job_id=jid, timeout_seconds=25, recovered=True)
# COMMIT -> publish gap and XADD -> outbox COMMIT gap, each crash is automatic.
for point in ('before_publish', 'after_publish'):
compose('stop', '--timeout', '3', 'publisher')
+88
View File
@@ -0,0 +1,88 @@
"""Локальные проверки реальных subprocess без Docker и внешних библиотек.
python -m unittest discover -s student-tests -p test_task_supervisor.py -v
"""
from pathlib import Path
import subprocess
import sys
from tempfile import TemporaryDirectory
import time
import unittest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src import task_supervisor
class TaskSupervisorTests(unittest.TestCase):
def test_successful_work_outlives_initial_lease(self):
lease = 0.5
expires = time.monotonic() + lease
renewals = []
def renew():
nonlocal expires
now = time.monotonic()
self.assertLess(now, expires, 'renewal must happen before lease expires')
expires = now + lease
renewals.append(now)
return True
started = time.monotonic()
code = task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(1.1)'],
renew, timeout=5, interval=0.1)
self.assertEqual(code, 0)
self.assertGreater(time.monotonic() - started, lease)
self.assertGreaterEqual(len(renewals), 3)
def test_heartbeat_does_not_bypass_total_timeout(self):
renewals = []
def renew():
renewals.append(True)
return True
started = time.monotonic()
with self.assertRaises(subprocess.TimeoutExpired):
task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'],
renew, timeout=0.5, interval=0.1)
self.assertTrue(renewals)
self.assertLess(time.monotonic() - started, 5)
def test_lost_lease_stops_work(self):
with self.assertRaises(task_supervisor.LeaseLost):
task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'],
lambda: False, timeout=10, interval=0.1)
def test_renewal_error_stops_work(self):
def renew():
raise ConnectionError('database unavailable')
with self.assertRaises(ConnectionError):
task_supervisor.run([sys.executable, '-c', 'import time; time.sleep(60)'],
renew, timeout=10, interval=0.1)
def test_child_failure_is_not_success(self):
self.assertEqual(task_supervisor.run([sys.executable, '-c', 'raise SystemExit(86)'],
lambda: True, timeout=5, interval=0.1), 86)
def test_timeout_kills_descendant_too(self):
with TemporaryDirectory() as directory:
marker = Path(directory) / 'survived'
started = Path(directory) / 'started'
descendant = ('import time; from pathlib import Path; '
f'Path({str(started)!r}).write_text("started"); '
'time.sleep(1.5); '
f'Path({str(marker)!r}).write_text("orphan")')
parent = ('import subprocess,sys,time; '
f'subprocess.Popen([sys.executable,"-c",{descendant!r}]); '
'time.sleep(60)')
with self.assertRaises(subprocess.TimeoutExpired):
task_supervisor.run([sys.executable, '-c', parent], lambda: True,
timeout=0.8, interval=0.1)
self.assertTrue(started.exists(), 'descendant must start before timeout')
time.sleep(1.6)
self.assertFalse(marker.exists(), 'descendant survived parent timeout')
if __name__ == '__main__':
unittest.main()