forked from mtusi2026/mtusi2026lab5
96 lines
5.0 KiB
Python
96 lines
5.0 KiB
Python
"""Транзакционные переходы состояний, bounded retry и явный audit replay."""
|
|
import os
|
|
import random
|
|
from uuid import UUID, uuid4
|
|
|
|
from src import db
|
|
|
|
MAX_ATTEMPTS = int(os.environ.get('JOB_MAX_ATTEMPTS', '5'))
|
|
TIMEOUT = int(os.environ.get('JOB_TIMEOUT_SECONDS', '60'))
|
|
LEASE = int(os.environ.get('JOB_LEASE_SECONDS', '90'))
|
|
BACKOFF = float(os.environ.get('JOB_BACKOFF_SECONDS', '2'))
|
|
if not 1 <= MAX_ATTEMPTS <= 20 or TIMEOUT < 1 or LEASE < TIMEOUT + 5 or BACKOFF <= 0:
|
|
raise ValueError('Неверные retry/timeout/lease настройки')
|
|
|
|
|
|
def event(conn, job, name, error=None):
|
|
conn.execute('INSERT INTO attachment_job_events(job_id,attempt,event,error_code) VALUES (%s,%s,%s,%s)',
|
|
(job['id'], job['attempt'], name, error))
|
|
|
|
|
|
def locked(conn, job_id):
|
|
return conn.execute('''SELECT j.*, a.status, a.object_key, a.variants, a.sha256, a.size_bytes,
|
|
a.content_type FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
|
WHERE j.id=%s FOR UPDATE OF j,a''', (job_id,)).fetchone()
|
|
|
|
|
|
def claim(payload):
|
|
with db.connection() as conn:
|
|
job = locked(conn, UUID(payload['job_id']))
|
|
if not job or str(job['attachment_id']) != payload['attachment_id'] or job['pipeline_version'] != payload['pipeline_version']:
|
|
return 'obsolete', None
|
|
if job['status'] in {'ready', 'failed'}:
|
|
return job['status'], job
|
|
if payload['attempt'] != job['attempt']:
|
|
return 'obsolete', None
|
|
if job['status'] != 'queued':
|
|
return 'busy', None
|
|
row = conn.execute('''UPDATE attachment_jobs SET lease_token=%s,
|
|
lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s
|
|
AND next_attempt_at<=clock_timestamp() RETURNING *''', (uuid4(), LEASE, job['id'])).fetchone()
|
|
if not row:
|
|
return 'busy', None
|
|
job.update(row)
|
|
conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],))
|
|
event(conn, job, 'processing')
|
|
return 'claimed', job
|
|
|
|
|
|
def retry_or_fail(conn, job, code, permanent=False):
|
|
# replay_count задаёт новый ограниченный бюджет, сохраняя глобальный номер attempt.
|
|
exhausted = job['attempt'] >= job['max_attempts']
|
|
if permanent or exhausted:
|
|
final_code = code if permanent else 'retry_exhausted'
|
|
conn.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
|
|
(final_code, job['attachment_id']))
|
|
conn.execute('UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL,last_error=%s WHERE id=%s', (code, job['id']))
|
|
event(conn, job, 'failed', final_code)
|
|
return
|
|
delay = random.uniform(0.5, 1.5) * min(60, BACKOFF * 2 ** min(job['attempt'] - 1, 10))
|
|
conn.execute('''UPDATE attachment_jobs SET attempt=attempt+1, lease_token=NULL,lease_until=NULL,
|
|
last_error=%s,next_attempt_at=clock_timestamp()+%s*interval '1 second' WHERE id=%s''', (code, delay, job['id']))
|
|
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],))
|
|
conn.execute('''INSERT INTO attachment_outbox(job_id,attempt,available_at)
|
|
SELECT id,attempt,next_attempt_at FROM attachment_jobs WHERE id=%s''', (job['id'],))
|
|
event(conn, job, 'retry', code)
|
|
|
|
|
|
def fail(job, code, permanent=False):
|
|
with db.connection() as conn:
|
|
current = locked(conn, job['id'])
|
|
if current['status'] == 'processing' and current['lease_token'] == job['lease_token']:
|
|
retry_or_fail(conn, current, code, permanent)
|
|
|
|
|
|
def recover():
|
|
with db.connection() as conn:
|
|
rows = conn.execute('''SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id
|
|
WHERE a.status='processing' AND j.lease_until<clock_timestamp()
|
|
ORDER BY j.lease_until LIMIT 20 FOR UPDATE OF j SKIP LOCKED''').fetchall()
|
|
for row in rows:
|
|
job = locked(conn, row['id'])
|
|
retry_or_fail(conn, job, 'worker_lost')
|
|
|
|
|
|
def replay(job_id):
|
|
with db.connection() as conn:
|
|
job = locked(conn, UUID(job_id))
|
|
if not job or job['status'] != 'failed' or job['staging_deleted_at'] is not None:
|
|
raise ValueError('Replay разрешён только для failed с сохранённым staging')
|
|
conn.execute('''UPDATE attachment_jobs SET attempt=attempt+1,replay_count=replay_count+1,
|
|
max_attempts=attempt+%s,next_attempt_at=clock_timestamp(),last_error=NULL,
|
|
lease_token=NULL,lease_until=NULL WHERE id=%s''', (MAX_ATTEMPTS, job['id']))
|
|
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", (job['attachment_id'],))
|
|
conn.execute('INSERT INTO attachment_outbox(job_id,attempt) SELECT id,attempt FROM attachment_jobs WHERE id=%s', (job['id'],))
|
|
event(conn, job, 'operator_replay')
|