"""Fenced, serial worker; DB row locks protect all effects of one job/version.""" import hashlib import json import os from pathlib import Path import random import signal import subprocess import sys import tempfile import threading import time import uuid from datetime import datetime, timezone from botocore.exceptions import BotoCoreError, ClientError from psycopg.rows import dict_row import redis from app import store, object_store from pipeline import audit, broker, fetch_job, GROUP, LEASE, MAX_ATTEMPTS, STREAM, TIMEOUT stop=threading.Event() class Permanent(Exception): pass def failpoint(name, selected): # CLI/in-process test only, never from broker or HTTP. if name == selected: os._exit(86) def valid_message(msg): required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'} if not isinstance(msg,dict) or not required<=msg.keys() or set(msg)-required-{'traceparent','tracestate'}: return False try: uuid.UUID(msg['job_id']);uuid.UUID(msg['attachment_id']) datetime.fromisoformat(msg['created_at'].replace('Z','+00:00')) return (type(msg['schema_version']) is int and msg['schema_version']==1 and msg['type']=='attachment.process.v1' and type(msg['attempt']) is int and msg['attempt']>=1 and type(msg['pipeline_version']) is int and msg['pipeline_version']>=1) except (ValueError,TypeError,AttributeError):return False def claim(msg): with store.connect() as conn: job=fetch_job(conn,msg['job_id'],'FOR UPDATE SKIP LOCKED') if job is None:return 'defer',None if str(job['attachment_id'])!=msg['attachment_id'] or job['pipeline_version']!=msg['pipeline_version']: return 'invalid',None if job['state'] in ('ready','failed','superseded'): audit(conn,job['id'],'duplicate_noop') return 'noop',job att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone() if att['pipeline_version']!=job['pipeline_version']: conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],)) audit(conn,job['id'],'stale_noop');return 'noop',job now=datetime.now(timezone.utc) if job['next_at']>now or (job['state']=='processing' and job['lease_until'] and job['lease_until']>now): return 'defer',job if job['attempt']>=MAX_ATTEMPTS: conn.execute("UPDATE jobs SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(job['id'],)) conn.execute("UPDATE attachments SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(att['id'],)) audit(conn,job['id'],'retry_exhausted');return 'noop',job job=conn.cursor(row_factory=dict_row).execute("""UPDATE jobs SET state='processing',attempt=attempt+1, lease_until=now()+(%s*interval '1 second'),updated_at=now() WHERE id=%s RETURNING *""",(LEASE,job['id'])).fetchone() conn.execute("UPDATE attachments SET state='processing',updated_at=now() WHERE id=%s",(att['id'],)) audit(conn,job['id'],'claimed') return 'claimed',job def record_error(job, code, permanent): with store.connect() as conn: current=fetch_job(conn,job['id'],'FOR UPDATE') if current['state']!='processing' or current['attempt']!=job['attempt']:return final=permanent or current['attempt']>=MAX_ATTEMPTS state='failed' if final else 'queued' code=code if permanent or not final else 'retry_exhausted' delay=random.uniform(.5,1.5)*min(30,2**(current['attempt']-1)) conn.execute("UPDATE jobs SET state=%s,error_code=%s,next_at=now()+(%s*interval '1 second'),lease_until=NULL,updated_at=now() WHERE id=%s",(state,code,delay,job['id'])) conn.execute("UPDATE attachments SET state=%s,error_code=%s,updated_at=now() WHERE id=%s AND pipeline_version=%s", (state,code if final else None,job['attachment_id'],job['pipeline_version'])) conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],)) audit(conn,job['id'],'failed:'+code if final else 'retry:'+code) def handle(msg, crash=None, force_error=None): if not valid_message(msg):return 'invalid' action,job=claim(msg) if action!='claimed':return action try: with store.connect() as conn: current=fetch_job(conn,job['id'],'FOR UPDATE') if current['state']!='processing' or current['attempt']!=job['attempt']:return 'defer' att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone() if att['pipeline_version']!=job['pipeline_version']: conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],));return 'noop' if force_error=='permanent':raise Permanent('invalid_image') if force_error=='temporary':raise TimeoutError() failpoint('before_effect',crash) with tempfile.TemporaryDirectory() as folder: root=Path(folder) response=object_store.client.get_object(Bucket=object_store.bucket,Key=att['staging_key']) try:payload=response['Body'].read(10485761) finally:response['Body'].close() if len(payload)!=att['size_bytes'] or hashlib.sha256(payload).hexdigest()!=att['sha256']: raise Permanent('staging_corrupt') (root/'input').write_bytes(payload) result=subprocess.run([sys.executable,'transform.py',str(root/'input'),str(root),str(job['pipeline_version'])],timeout=TIMEOUT, env={'PATH':os.environ.get('PATH','/usr/local/bin:/usr/bin:/bin'),'LANG':'C.UTF-8'}, stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False) if result.returncode: raise Permanent('resource_limit' if result.returncode in (3,-9) else 'invalid_image') variants=json.loads((root/'manifest.json').read_text()) prefix=f"results/{att['id']}/v{job['pipeline_version']}/" key=prefix+'original' def put(key,data,mime): object_store.client.put_object(Bucket=object_store.bucket,Key=key,Body=data, ContentType=mime,Metadata={'sha256':hashlib.sha256(data).hexdigest()}) put(key,payload,att['content_type']) for variant in variants: variant['object_key']=prefix+variant['name'] put(variant['object_key'],(root/variant['name']).read_bytes(),variant['content_type']) failpoint('after_effect',crash) conn.execute("""UPDATE attachments SET state='ready',error_code=NULL,object_key=%s, variants=%s::jsonb,updated_at=now() WHERE id=%s AND pipeline_version=%s""", (key,json.dumps(variants),att['id'],job['pipeline_version'])) conn.execute("UPDATE jobs SET state='ready',error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=%s",(job['id'],)) audit(conn,job['id'],'ready') failpoint('after_commit',crash) return 'ready' except Permanent as exc: record_error(job,str(exc),True);return 'failed' except ClientError as exc: code=exc.response.get('Error',{}).get('Code','') record_error(job,'staging_missing' if code in ('NoSuchKey','404') else 'storage_unavailable',code in ('NoSuchKey','404')) return 'retry' except (BotoCoreError,TimeoutError,subprocess.TimeoutExpired): record_error(job,'processing_timeout',False);return 'retry' except Exception: # Includes a lost/ambiguous DB COMMIT: never delete effects. Lease and # durable outbox safely recover; a ready duplicate cannot be rerun. print(json.dumps({'worker_error':'infrastructure','job_id':str(job['id'])}),flush=True) return 'defer' def ack(client,mid): client.xack(STREAM,GROUP,mid) client.xdel(STREAM,mid) def main(): store.pool.open(wait=True,timeout=30) client=broker();consumer=str(uuid.uuid4()) for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:stop.set()) try: while not stop.is_set(): try: try:client.xgroup_create(STREAM,GROUP,id='0',mkstream=True) except redis.ResponseError as exc: if 'BUSYGROUP' not in str(exc):raise reclaimed=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE*1000,start_id='0-0',count=1)[1] entries=reclaimed if not entries: batches=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000) entries=batches[0][1] if batches else [] for mid,fields in entries: try:msg=json.loads(fields.get('job','null')) except ValueError:msg=None outcome=handle(msg) # A failed attempt has a durable retry intent/failed record. # Defer leaves the delivery pending for lease-based reclaim. if outcome!='defer':ack(client,mid) print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True) except Exception as exc: print(json.dumps({'worker_error':type(exc).__name__}),flush=True);stop.wait(1) finally: client.close();object_store.close();store.pool.close() if __name__=='__main__':main()