Files
mtusi2026lab5/worker.py
T

184 lines
9.5 KiB
Python

"""Fenced, serial worker; DB row locks protect all effects of one job/version."""
import hashlib
import json
import os
from pathlib import Path
import random
import signal
import subprocess
import sys
import tempfile
import threading
import time
import uuid
from datetime import datetime, timezone
from botocore.exceptions import BotoCoreError, ClientError
from psycopg.rows import dict_row
import redis
from app import store, object_store
from pipeline import audit, broker, fetch_job, GROUP, LEASE, MAX_ATTEMPTS, STREAM, TIMEOUT
stop=threading.Event()
class Permanent(Exception):
pass
def failpoint(name, selected):
# CLI/in-process test only, never from broker or HTTP.
if name == selected:
os._exit(86)
def valid_message(msg):
required={'schema_version','job_id','type','attachment_id','pipeline_version','attempt','created_at'}
if not isinstance(msg,dict) or not required<=msg.keys() or set(msg)-required-{'traceparent','tracestate'}:
return False
try:
uuid.UUID(msg['job_id']);uuid.UUID(msg['attachment_id'])
datetime.fromisoformat(msg['created_at'].replace('Z','+00:00'))
return (type(msg['schema_version']) is int and msg['schema_version']==1 and msg['type']=='attachment.process.v1'
and type(msg['attempt']) is int and msg['attempt']>=1
and type(msg['pipeline_version']) is int and msg['pipeline_version']>=1)
except (ValueError,TypeError,AttributeError):return False
def claim(msg):
with store.connect() as conn:
job=fetch_job(conn,msg['job_id'],'FOR UPDATE SKIP LOCKED')
if job is None:return 'defer',None
if str(job['attachment_id'])!=msg['attachment_id'] or job['pipeline_version']!=msg['pipeline_version']:
return 'invalid',None
if job['state'] in ('ready','failed','superseded'):
audit(conn,job['id'],'duplicate_noop')
return 'noop',job
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
if att['pipeline_version']!=job['pipeline_version']:
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],))
audit(conn,job['id'],'stale_noop');return 'noop',job
now=datetime.now(timezone.utc)
if job['next_at']>now or (job['state']=='processing' and job['lease_until'] and job['lease_until']>now):
return 'defer',job
if job['attempt']>=MAX_ATTEMPTS:
conn.execute("UPDATE jobs SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(job['id'],))
conn.execute("UPDATE attachments SET state='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s",(att['id'],))
audit(conn,job['id'],'retry_exhausted');return 'noop',job
job=conn.cursor(row_factory=dict_row).execute("""UPDATE jobs SET state='processing',attempt=attempt+1,
lease_until=now()+(%s*interval '1 second'),updated_at=now() WHERE id=%s RETURNING *""",(LEASE,job['id'])).fetchone()
conn.execute("UPDATE attachments SET state='processing',updated_at=now() WHERE id=%s",(att['id'],))
audit(conn,job['id'],'claimed')
return 'claimed',job
def record_error(job, code, permanent):
with store.connect() as conn:
current=fetch_job(conn,job['id'],'FOR UPDATE')
if current['state']!='processing' or current['attempt']!=job['attempt']:return
final=permanent or current['attempt']>=MAX_ATTEMPTS
state='failed' if final else 'queued'
code=code if permanent or not final else 'retry_exhausted'
delay=random.uniform(.5,1.5)*min(30,2**(current['attempt']-1))
conn.execute("UPDATE jobs SET state=%s,error_code=%s,next_at=now()+(%s*interval '1 second'),lease_until=NULL,updated_at=now() WHERE id=%s",(state,code,delay,job['id']))
conn.execute("UPDATE attachments SET state=%s,error_code=%s,updated_at=now() WHERE id=%s AND pipeline_version=%s",
(state,code if final else None,job['attachment_id'],job['pipeline_version']))
conn.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job['id'],))
audit(conn,job['id'],'failed:'+code if final else 'retry:'+code)
def handle(msg, crash=None, force_error=None):
if not valid_message(msg):return 'invalid'
action,job=claim(msg)
if action!='claimed':return action
try:
with store.connect() as conn:
current=fetch_job(conn,job['id'],'FOR UPDATE')
if current['state']!='processing' or current['attempt']!=job['attempt']:return 'defer'
att=conn.cursor(row_factory=dict_row).execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
if att['pipeline_version']!=job['pipeline_version']:
conn.execute("UPDATE jobs SET state='superseded' WHERE id=%s",(job['id'],));return 'noop'
if force_error=='permanent':raise Permanent('invalid_image')
if force_error=='temporary':raise TimeoutError()
failpoint('before_effect',crash)
with tempfile.TemporaryDirectory() as folder:
root=Path(folder)
response=object_store.client.get_object(Bucket=object_store.bucket,Key=att['staging_key'])
try:payload=response['Body'].read(10485761)
finally:response['Body'].close()
if len(payload)!=att['size_bytes'] or hashlib.sha256(payload).hexdigest()!=att['sha256']:
raise Permanent('staging_corrupt')
(root/'input').write_bytes(payload)
result=subprocess.run([sys.executable,'transform.py',str(root/'input'),str(root),str(job['pipeline_version'])],timeout=TIMEOUT,
env={'PATH':os.environ.get('PATH','/usr/local/bin:/usr/bin:/bin'),'LANG':'C.UTF-8'},
stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=False)
if result.returncode:
raise Permanent('resource_limit' if result.returncode in (3,-9) else 'invalid_image')
variants=json.loads((root/'manifest.json').read_text())
prefix=f"results/{att['id']}/v{job['pipeline_version']}/"
key=prefix+'original'
def put(key,data,mime):
object_store.client.put_object(Bucket=object_store.bucket,Key=key,Body=data,
ContentType=mime,Metadata={'sha256':hashlib.sha256(data).hexdigest()})
put(key,payload,att['content_type'])
for variant in variants:
variant['object_key']=prefix+variant['name']
put(variant['object_key'],(root/variant['name']).read_bytes(),variant['content_type'])
failpoint('after_effect',crash)
conn.execute("""UPDATE attachments SET state='ready',error_code=NULL,object_key=%s,
variants=%s::jsonb,updated_at=now() WHERE id=%s AND pipeline_version=%s""",
(key,json.dumps(variants),att['id'],job['pipeline_version']))
conn.execute("UPDATE jobs SET state='ready',error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=%s",(job['id'],))
audit(conn,job['id'],'ready')
failpoint('after_commit',crash)
return 'ready'
except Permanent as exc:
record_error(job,str(exc),True);return 'failed'
except ClientError as exc:
code=exc.response.get('Error',{}).get('Code','')
record_error(job,'staging_missing' if code in ('NoSuchKey','404') else 'storage_unavailable',code in ('NoSuchKey','404'))
return 'retry'
except (BotoCoreError,TimeoutError,subprocess.TimeoutExpired):
record_error(job,'processing_timeout',False);return 'retry'
except Exception:
# Includes a lost/ambiguous DB COMMIT: never delete effects. Lease and
# durable outbox safely recover; a ready duplicate cannot be rerun.
print(json.dumps({'worker_error':'infrastructure','job_id':str(job['id'])}),flush=True)
return 'defer'
def ack(client,mid):
client.xack(STREAM,GROUP,mid)
client.xdel(STREAM,mid)
def main():
store.pool.open(wait=True,timeout=30)
client=broker();consumer=str(uuid.uuid4())
for sig in (signal.SIGTERM,signal.SIGINT):signal.signal(sig,lambda *_:stop.set())
try:
while not stop.is_set():
try:
try:client.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):raise
reclaimed=client.xautoclaim(STREAM,GROUP,consumer,min_idle_time=LEASE*1000,start_id='0-0',count=1)[1]
entries=reclaimed
if not entries:
batches=client.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
entries=batches[0][1] if batches else []
for mid,fields in entries:
try:msg=json.loads(fields.get('job','null'))
except ValueError:msg=None
outcome=handle(msg)
# A failed attempt has a durable retry intent/failed record.
# Defer leaves the delivery pending for lease-based reclaim.
if outcome!='defer':ack(client,mid)
print(json.dumps({'outcome':outcome,'delivery_id':mid}),flush=True)
except Exception as exc:
print(json.dumps({'worker_error':type(exc).__name__}),flush=True);stop.wait(1)
finally:
client.close();object_store.close();store.pool.close()
if __name__=='__main__':main()