forked from mtusi2026/mtusi2026lab6
141 lines
6.4 KiB
Python
141 lines
6.4 KiB
Python
"""Transactional claims, bounded retries and explicit failed-job recovery."""
|
|
import os
|
|
import random
|
|
import datetime as dt
|
|
|
|
from . import telemetry
|
|
|
|
from .errors import Conflict, NotFound
|
|
|
|
LEASE_SECONDS = 60
|
|
STAGING_RETENTION_SECONDS = 86400
|
|
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
|
if not 1 <= MAX_ATTEMPTS <= 20:
|
|
raise ValueError('JOB_MAX_ATTEMPTS must be between 1 and 20')
|
|
|
|
|
|
def _locked_job(cur, job_id):
|
|
cur.execute(
|
|
'SELECT j.*, a.status, a.updated_at, a.error_code, '
|
|
'j.lease_until > clock_timestamp() AS lease_valid FROM attachment_jobs j '
|
|
'JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s FOR UPDATE OF j,a',
|
|
(job_id,))
|
|
return cur.fetchone()
|
|
|
|
|
|
def claim(db, delivery):
|
|
with db.cursor() as cur:
|
|
row = _locked_job(cur, delivery['job_id'])
|
|
if (not row or row['status'] != 'queued' or row['attempt'] != delivery['attempt']
|
|
or str(row['attachment_id']) != delivery['attachment_id']
|
|
or row['pipeline_version'] != delivery['pipeline_version']):
|
|
return None
|
|
cur.execute('UPDATE attachment_jobs SET lease_until=clock_timestamp()+make_interval(secs => %s), '
|
|
'attempts_used=attempts_used+1 WHERE id=%s', (LEASE_SECONDS, row['id']))
|
|
cur.execute("UPDATE attachments SET status='processing', updated_at=now() "
|
|
"WHERE id=%s RETURNING *", (row['attachment_id'],))
|
|
return cur.fetchone()
|
|
|
|
|
|
def _requeue(cur, row, delay, reset=False):
|
|
attempt = row['attempt'] + 1
|
|
cur.execute('UPDATE attachment_jobs SET attempt=%s, lease_until=NULL, attempts_used=%s WHERE id=%s',
|
|
(attempt, 0 if reset else row['attempts_used'], row['id']))
|
|
cur.execute("UPDATE attachments SET status='queued', error_code=NULL, updated_at=now() WHERE id=%s",
|
|
(row['attachment_id'],))
|
|
cur.execute('INSERT INTO attachment_outbox (job_id,attempt,available_at) '
|
|
'VALUES (%s,%s,now()+make_interval(secs => %s))', (row['id'], attempt, delay))
|
|
return attempt
|
|
|
|
|
|
def _fail_or_retry(cur, row, error_code):
|
|
if error_code is None and row['attempts_used'] < MAX_ATTEMPTS:
|
|
delay = min(60, 2 ** row['attempts_used']) * random.uniform(.75, 1.25)
|
|
_requeue(cur, row, delay)
|
|
return
|
|
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
|
|
cur.execute("UPDATE attachments SET status='failed', error_code=%s, updated_at=now() WHERE id=%s",
|
|
(error_code or 'retry_exhausted', row['attachment_id']))
|
|
|
|
|
|
def complete(db, delivery, result=None, error_code=None):
|
|
with db.cursor() as cur:
|
|
row = _locked_job(cur, delivery['job_id'])
|
|
if (not row or row['status'] != 'processing'
|
|
or row['attempt'] != delivery['attempt'] or not row['lease_valid']):
|
|
return False
|
|
if result is None:
|
|
_fail_or_retry(cur, row, error_code)
|
|
return True
|
|
key, variants = result
|
|
cur.executemany(
|
|
'INSERT INTO attachment_variants (attachment_id,pipeline_version,variant,object_key,'
|
|
'content_type,size_bytes,sha256,width,height) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)',
|
|
[(row['attachment_id'], row['pipeline_version'], *item) for item in variants])
|
|
cur.execute("UPDATE attachments SET object_key=%s,status='ready',updated_at=now() WHERE id=%s",
|
|
(key, row['attachment_id']))
|
|
cur.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s', (row['id'],))
|
|
elapsed = (dt.datetime.now(dt.timezone.utc) - row['created_at']).total_seconds()
|
|
telemetry.time_to_ready.record(max(0, elapsed), {'job_type': 'attachment.process.v1'})
|
|
return True
|
|
|
|
|
|
def recover_expired(db):
|
|
with db.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
|
"WHERE a.status='processing' AND j.lease_until <= now() "
|
|
"ORDER BY j.lease_until LIMIT 10 FOR UPDATE OF j,a SKIP LOCKED")
|
|
for row in cur.fetchall():
|
|
_fail_or_retry(cur, row, None)
|
|
|
|
|
|
def replay(db, job_id, operator):
|
|
with db.cursor() as cur:
|
|
row = _locked_job(cur, job_id)
|
|
if row is None:
|
|
raise NotFound('Задача не найдена')
|
|
if row['status'] != 'failed' or row['staging_cleaned_at'] is not None:
|
|
raise Conflict('Повтор доступен только для failed с сохранённым исходником')
|
|
attempt = _requeue(cur, row, 0, reset=True)
|
|
cur.execute('INSERT INTO attachment_replays (job_id,attempt,operator) VALUES (%s,%s,%s)',
|
|
(job_id, attempt, operator))
|
|
return attempt
|
|
|
|
|
|
def clean_staging(db, remove):
|
|
with db.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT j.id,j.attachment_id FROM attachment_jobs j "
|
|
"JOIN attachments a ON a.id=j.attachment_id "
|
|
"WHERE j.staging_cleaned_at IS NULL AND (a.status='ready' OR "
|
|
"(a.status='failed' AND a.updated_at < now()-make_interval(secs => %s))) "
|
|
"LIMIT 1 FOR UPDATE OF j,a SKIP LOCKED", (STAGING_RETENTION_SECONDS,))
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return False
|
|
remove('attachments/staging/' + str(row['attachment_id']))
|
|
cur.execute('UPDATE attachment_jobs SET staging_cleaned_at=now() WHERE id=%s', (row['id'],))
|
|
return True
|
|
|
|
|
|
def clean_abandoned_outputs(db, remove):
|
|
with db.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT j.id,j.attachment_id,j.pipeline_version,o.attempt "
|
|
"FROM attachment_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
|
"JOIN attachments a ON a.id=j.attachment_id "
|
|
"WHERE o.objects_cleaned_at IS NULL AND o.available_at < now()-interval '15 minutes' "
|
|
"AND (o.attempt < j.attempt OR (a.status='failed' AND "
|
|
"a.updated_at < now()-interval '15 minutes')) "
|
|
"ORDER BY o.available_at LIMIT 1 FOR UPDATE OF o SKIP LOCKED")
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return False
|
|
prefix = 'attachments/final/{attachment_id}/v{pipeline_version}/a{attempt}/'.format(**row)
|
|
for name in ('original', 'thumbnail.jpeg', 'crop.jpeg', 'optimized.webp'):
|
|
remove(prefix + name)
|
|
cur.execute('UPDATE attachment_outbox SET objects_cleaned_at=now() WHERE job_id=%s AND attempt=%s',
|
|
(row['id'], row['attempt']))
|
|
return True
|