Files

69 lines
3.4 KiB
Python

"""Database authority, version fencing and safe operator actions."""
import json
import os
import uuid
from psycopg.rows import dict_row
import redis
STREAM = 'attachments.v1'
GROUP = 'image-workers'
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '20')))
LEASE = max(TIMEOUT + 5, int(os.getenv('JOB_LEASE_SECONDS', '30')))
def broker():
return redis.Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
decode_responses=True, socket_connect_timeout=1, socket_timeout=3)
def create_job(conn, aid, version):
jid = str(uuid.uuid4())
conn.execute("INSERT INTO jobs(id,attachment_id,pipeline_version,state) VALUES (%s,%s,%s,'queued')", (jid,aid,version))
conn.execute('INSERT INTO outbox(job_id) VALUES (%s)', (jid,))
audit(conn, jid, 'accepted')
return jid
def audit(conn, jid, event):
conn.execute('INSERT INTO job_audit(job_id,event) VALUES (%s,%s)', (jid,event))
def envelope(row):
return dict(schema_version=1,job_id=str(row['id']),type='attachment.process.v1',
attachment_id=str(row['attachment_id']),pipeline_version=row['pipeline_version'],
attempt=row['attempt']+1,created_at=row['created_at'].isoformat().replace('+00:00','Z'))
def fetch_job(conn, jid, suffix=''):
return conn.cursor(row_factory=dict_row).execute('SELECT * FROM jobs WHERE id=%s '+suffix, (jid,)).fetchone()
def replay(store, jid):
with store.connect() as conn:
job = fetch_job(conn,jid,'FOR UPDATE')
if job is None:
raise ValueError('unknown job')
att = conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE', (job['attachment_id'],)).fetchone()
if job['state'] != 'failed' or att['pipeline_version'] != job['pipeline_version'] or not att['staging_key']:
raise ValueError('only current failed job with retained staging can be replayed')
conn.execute("UPDATE jobs SET state='queued',attempt=0,error_code=NULL,next_at=now(),lease_until=NULL WHERE id=%s",(jid,))
conn.execute("UPDATE attachments SET state='queued',error_code=NULL,updated_at=now() WHERE id=%s",(att['id'],))
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(jid,))
audit(conn,jid,'operator_replay')
def upgrade(store, aid):
# Only trusted CLI; never accept version/storage paths from HTTP clients.
with store.connect() as conn:
# Lock jobs before attachment, same order as the worker.
conn.execute('SELECT id FROM jobs WHERE attachment_id=%s ORDER BY pipeline_version FOR UPDATE',(aid,)).fetchall()
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(aid,)).fetchone()
if att is None or (att['state'] != 'ready' and not att['staging_key']):
raise ValueError('upgrade requires durable source')
version=att['pipeline_version']+1
# A completed immutable original is itself a durable source for upgrade.
conn.execute("UPDATE attachments SET pipeline_version=%s,staging_key=CASE WHEN state='ready' THEN object_key ELSE staging_key END,state='queued',variants='[]',error_code=NULL,updated_at=now() WHERE id=%s",(version,aid))
conn.execute("UPDATE jobs SET state='superseded',updated_at=now() WHERE attachment_id=%s AND state IN ('queued','processing','failed')",(aid,))
return create_job(conn,aid,version)