184 lines
9.5 KiB
Python
184 lines
9.5 KiB
Python
"""Fenced, serial worker; DB row locks protect all effects of one job/version."""
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from datetime import datetime, timezone
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from psycopg.rows import dict_row
|
|
import redis
|
|
from app import store, object_store
|
|
from pipeline import audit, broker, fetch_job, GROUP, LEASE, MAX_ATTEMPTS, STREAM, TIMEOUT
|
|
|
|
stop=threading.Event()
|
|
|
|
|
|
class Permanent(Exception):
|
|
pass
|
|
|
|
|
|
def failpoint(name, selected):
|
|
# CLI/in-process test only, never from broker or HTTP.
|
|
if name == selected:
|
|
os._exit(86)
|
|
|
|
|
|
def valid_message(msg):
|
|
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
|
|
if not isinstance(msg,dict) or not required<=msg.keys() or set(msg)-required-{'traceparent','tracestate'}:
|
|
return False
|
|
try:
|
|
uuid.UUID(msg['job_id']);uuid.UUID(msg['attachment_id'])
|
|
datetime.fromisoformat(msg['created_at'].replace('Z','+00:00'))
|
|
return (type(msg['schema_version']) is int and msg['schema_version']==1 and msg['type']=='attachment.process.v1'
|
|
and type(msg['attempt']) is int and msg['attempt']>=1
|
|
and type(msg['pipeline_version']) is int and msg['pipeline_version']>=1)
|
|
except (ValueError,TypeError,AttributeError):return False
|
|
|
|
|
|
def claim(msg):
|
|
with store.connect() as conn:
|
|
job=fetch_job(conn,msg['job_id'],'FOR UPDATE SKIP LOCKED')
|
|
if job is None:return 'defer',None
|
|
if str(job['attachment_id'])!=msg['attachment_id'] or job['pipeline_version']!=msg['pipeline_version']:
|
|
return 'invalid',None
|
|
if job['state'] in ('ready','failed','superseded'):
|
|
audit(conn,job['id'],'duplicate_noop')
|
|
return 'noop',job
|
|
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
|
|
if att['pipeline_version']!=job['pipeline_version']:
|
|
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],))
|
|
audit(conn,job['id'],'stale_noop');return 'noop',job
|
|
now=datetime.now(timezone.utc)
|
|
if job['next_at']>now or (job['state']=='processing' and job['lease_until'] and job['lease_until']>now):
|
|
return 'defer',job
|
|
if job['attempt']>=MAX_ATTEMPTS:
|
|
conn.execute("UPDATE jobs SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(job['id'],))
|
|
conn.execute("UPDATE attachments SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(att['id'],))
|
|
audit(conn,job['id'],'retry_exhausted');return 'noop',job
|
|
job=conn.cursor(row_factory=dict_row).execute("""UPDATE jobs SET state='processing',attempt=attempt+1,
|
|
lease_until=now()+(%s*interval '1 second'),updated_at=now() WHERE id=%s RETURNING *""",(LEASE,job['id'])).fetchone()
|
|
conn.execute("UPDATE attachments SET state='processing',updated_at=now() WHERE id=%s",(att['id'],))
|
|
audit(conn,job['id'],'claimed')
|
|
return 'claimed',job
|
|
|
|
|
|
def record_error(job, code, permanent):
|
|
with store.connect() as conn:
|
|
current=fetch_job(conn,job['id'],'FOR UPDATE')
|
|
if current['state']!='processing' or current['attempt']!=job['attempt']:return
|
|
final=permanent or current['attempt']>=MAX_ATTEMPTS
|
|
state='failed' if final else 'queued'
|
|
code=code if permanent or not final else 'retry_exhausted'
|
|
delay=random.uniform(.5,1.5)*min(30,2**(current['attempt']-1))
|
|
conn.execute("UPDATE jobs SET state=%s,error_code=%s,next_at=now()+(%s*interval '1 second'),lease_until=NULL,updated_at=now() WHERE id=%s",(state,code,delay,job['id']))
|
|
conn.execute("UPDATE attachments SET state=%s,error_code=%s,updated_at=now() WHERE id=%s AND pipeline_version=%s",
|
|
(state,code if final else None,job['attachment_id'],job['pipeline_version']))
|
|
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],))
|
|
audit(conn,job['id'],'failed:'+code if final else 'retry:'+code)
|
|
|
|
|
|
def handle(msg, crash=None, force_error=None):
|
|
if not valid_message(msg):return 'invalid'
|
|
action,job=claim(msg)
|
|
if action!='claimed':return action
|
|
try:
|
|
with store.connect() as conn:
|
|
current=fetch_job(conn,job['id'],'FOR UPDATE')
|
|
if current['state']!='processing' or current['attempt']!=job['attempt']:return 'defer'
|
|
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
|
|
if att['pipeline_version']!=job['pipeline_version']:
|
|
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],));return 'noop'
|
|
if force_error=='permanent':raise Permanent('invalid_image')
|
|
if force_error=='temporary':raise TimeoutError()
|
|
failpoint('before_effect',crash)
|
|
with tempfile.TemporaryDirectory() as folder:
|
|
root=Path(folder)
|
|
response=object_store.client.get_object(Bucket=object_store.bucket,Key=att['staging_key'])
|
|
try:payload=response['Body'].read(10485761)
|
|
finally:response['Body'].close()
|
|
if len(payload)!=att['size_bytes'] or hashlib.sha256(payload).hexdigest()!=att['sha256']:
|
|
raise Permanent('staging_corrupt')
|
|
(root/'input').write_bytes(payload)
|
|
result=subprocess.run([sys.executable,'transform.py',str(root/'input'),str(root),str(job['pipeline_version'])],timeout=TIMEOUT,
|
|
env={'PATH':os.environ.get('PATH','/usr/local/bin:/usr/bin:/bin'),'LANG':'C.UTF-8'},
|
|
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False)
|
|
if result.returncode:
|
|
raise Permanent('resource_limit' if result.returncode in (3,-9) else 'invalid_image')
|
|
variants=json.loads((root/'manifest.json').read_text())
|
|
prefix=f"results/{att['id']}/v{job['pipeline_version']}/"
|
|
key=prefix+'original'
|
|
def put(key,data,mime):
|
|
object_store.client.put_object(Bucket=object_store.bucket,Key=key,Body=data,
|
|
ContentType=mime,Metadata={'sha256':hashlib.sha256(data).hexdigest()})
|
|
put(key,payload,att['content_type'])
|
|
for variant in variants:
|
|
variant['object_key']=prefix+variant['name']
|
|
put(variant['object_key'],(root/variant['name']).read_bytes(),variant['content_type'])
|
|
failpoint('after_effect',crash)
|
|
conn.execute("""UPDATE attachments SET state='ready',error_code=NULL,object_key=%s,
|
|
variants=%s::jsonb,updated_at=now() WHERE id=%s AND pipeline_version=%s""",
|
|
(key,json.dumps(variants),att['id'],job['pipeline_version']))
|
|
conn.execute("UPDATE jobs SET state='ready',error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=%s",(job['id'],))
|
|
audit(conn,job['id'],'ready')
|
|
failpoint('after_commit',crash)
|
|
return 'ready'
|
|
except Permanent as exc:
|
|
record_error(job,str(exc),True);return 'failed'
|
|
except ClientError as exc:
|
|
code=exc.response.get('Error',{}).get('Code','')
|
|
record_error(job,'staging_missing' if code in ('NoSuchKey','404') else 'storage_unavailable',code in ('NoSuchKey','404'))
|
|
return 'retry'
|
|
except (BotoCoreError,TimeoutError,subprocess.TimeoutExpired):
|
|
record_error(job,'processing_timeout',False);return 'retry'
|
|
except Exception:
|
|
# Includes a lost/ambiguous DB COMMIT: never delete effects. Lease and
|
|
# durable outbox safely recover; a ready duplicate cannot be rerun.
|
|
print(json.dumps({'worker_error':'infrastructure','job_id':str(job['id'])}),flush=True)
|
|
return 'defer'
|
|
|
|
|
|
def ack(client,mid):
|
|
client.xack(STREAM,GROUP,mid)
|
|
client.xdel(STREAM,mid)
|
|
|
|
|
|
def main():
|
|
store.pool.open(wait=True,timeout=30)
|
|
client=broker();consumer=str(uuid.uuid4())
|
|
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:stop.set())
|
|
try:
|
|
while not stop.is_set():
|
|
try:
|
|
try:client.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if 'BUSYGROUP' not in str(exc):raise
|
|
reclaimed=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE*1000,start_id='0-0',count=1)[1]
|
|
entries=reclaimed
|
|
if not entries:
|
|
batches=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
|
|
entries=batches[0][1] if batches else []
|
|
for mid,fields in entries:
|
|
try:msg=json.loads(fields.get('job','null'))
|
|
except ValueError:msg=None
|
|
outcome=handle(msg)
|
|
# A failed attempt has a durable retry intent/failed record.
|
|
# Defer leaves the delivery pending for lease-based reclaim.
|
|
if outcome!='defer':ack(client,mid)
|
|
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
|
|
except Exception as exc:
|
|
print(json.dumps({'worker_error':type(exc).__name__}),flush=True);stop.wait(1)
|
|
finally:
|
|
client.close();object_store.close();store.pool.close()
|
|
|
|
|
|
if __name__=='__main__':main()
|