Files
mtusi2026lab6-fork/src/messenger/job_lifecycle.py
T

141 lines
6.4 KiB
Python

"""Transactional claims, bounded retries and explicit failed-job recovery."""
import os
import random
import datetime as dt
from . import telemetry
from .errors import Conflict, NotFound
LEASE_SECONDS = 60
STAGING_RETENTION_SECONDS = 86400
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
if not 1 <= MAX_ATTEMPTS <= 20:
raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20')
def _locked_job(cur, job_id):
cur.execute(
'SELECT j.*, a.status, a.updated_at, a.error_code, '
'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j '
'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a',
(job_id,))
return cur.fetchone()
def claim(db, delivery):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt']
or str(row['attachment_id']) != delivery['attachment_id']
or row['pipeline_version'] != delivery['pipeline_version']):
return None
cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), '
'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id']))
cur.execute("UPDATE attachments SET status='processing', updated_at=now() "
"WHERE id=%s RETURNING *", (row['attachment_id'],))
return cur.fetchone()
def _requeue(cur, row, delay, reset=False):
attempt = row['attempt'] + 1
cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s',
(attempt, 0 if reset else row['attempts_used'], row['id']))
cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s",
(row['attachment_id'],))
cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) '
'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay))
return attempt
def _fail_or_retry(cur, row, error_code):
if error_code is None and row['attempts_used'] < MAX_ATTEMPTS:
delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25)
_requeue(cur, row, delay)
return
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s",
(error_code or 'retry_exhausted', row['attachment_id']))
def complete(db, delivery, result=None, error_code=None):
with db.cursor() as cur:
row = _locked_job(cur, delivery['job_id'])
if (not row or row['status'] != 'processing'
or row['attempt'] != delivery['attempt'] or not row['lease_valid']):
return False
if result is None:
_fail_or_retry(cur, row, error_code)
return True
key, variants = result
cur.executemany(
'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,'
'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
[(row['attachment_id'], row['pipeline_version'], *item) for item in variants])
cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s",
(key, row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
elapsed = (dt.datetime.now(dt.timezone.utc) - row['created_at']).total_seconds()
telemetry.time_to_ready.record(max(0, elapsed), {'job_type': 'attachment.process.v1'})
return True
def recover_expired(db):
with db.cursor() as cur:
cur.execute(
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='processing' AND j.lease_until <= now() "
"ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED")
for row in cur.fetchall():
_fail_or_retry(cur, row, None)
def replay(db, job_id, operator):
with db.cursor() as cur:
row = _locked_job(cur, job_id)
if row is None:
raise NotFound('Задача не найдена')
if row['status'] != 'failed' or row['staging_cleaned_at'] is not None:
raise Conflict('Повтор доступен только для failed с сохранённым исходником')
attempt = _requeue(cur, row, 0, reset=True)
cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)',
(job_id, attempt, operator))
return attempt
def clean_staging(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id FROM attachment_jobs j "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR "
"(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) "
"LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,))
row = cur.fetchone()
if not row:
return False
remove('attachments/staging/' + str(row['attachment_id']))
cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],))
return True
def clean_abandoned_outputs(db, remove):
with db.cursor() as cur:
cur.execute(
"SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt "
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"JOIN attachments a ON a.id=j.attachment_id "
"WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' "
"AND (o.attempt < j.attempt OR (a.status='failed' AND "
"a.updated_at < now()-interval '15 minutes')) "
"ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED")
row = cur.fetchone()
if not row:
return False
prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row)
for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'):
remove(prefix + name)
cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s',
(row['id'], row['attempt']))
return True