72 lines
4.1 KiB
Python
72 lines
4.1 KiB
Python
"""Local operator CLI: failed registry, audited replay and versioned reprocessing."""
|
|
import argparse
|
|
import json
|
|
import tempfile
|
|
from uuid import UUID, uuid4
|
|
|
|
from .jobs import audit, database
|
|
from .storage import MAX_BYTES, Storage
|
|
|
|
|
|
def replay(job_id):
|
|
storage=Storage()
|
|
try:
|
|
with database() as db, db.transaction():
|
|
job=db.execute('SELECT * FROM jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone()
|
|
if not job or job['status']!='failed': raise ValueError('Only failed jobs can be replayed')
|
|
att=db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(job['attachment_id'],)).fetchone()
|
|
if att['pipeline_version']!=job['pipeline_version'] or not att['staging_key']:
|
|
raise ValueError('Stale version or expired staging; replay refused')
|
|
storage.client.head_object(Bucket=storage.bucket,Key=att['staging_key'])
|
|
db.execute("UPDATE jobs SET status='queued',attempts=0,error_code=NULL,lease_token=NULL,lease_until=NULL,next_attempt_at=clock_timestamp(),updated_at=clock_timestamp() WHERE id=%s",(job_id,))
|
|
db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(att['id'],))
|
|
db.execute('UPDATE outbox SET published_at=NULL WHERE job_id=%s',(job_id,))
|
|
audit(db,job_id,'operator_replay',previous_attempts=job['attempts'])
|
|
return {'job_id':str(job_id),'status':'queued'}
|
|
finally: storage.client.close()
|
|
|
|
|
|
def upgrade(attachment_id,version):
|
|
if version not in (1,2): raise ValueError('Supported pipeline versions are 1 and 2')
|
|
storage=Storage()
|
|
try:
|
|
with database() as db, db.transaction():
|
|
att=db.execute('SELECT * FROM attachments WHERE id=%s FOR UPDATE',(attachment_id,)).fetchone()
|
|
if not att or version<=att['pipeline_version']: raise ValueError('Version must strictly increase')
|
|
source=att['object_key'] if att['status']=='ready' else att['staging_key']
|
|
if not source: raise ValueError('Original is no longer available')
|
|
key=f'staging/{attachment_id}/v{version}'
|
|
db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))',(key,))
|
|
obj=storage.get(source)
|
|
try:
|
|
with tempfile.TemporaryFile() as file:
|
|
size=0
|
|
while block:=obj['Body'].read(65536):
|
|
size+=len(block)
|
|
if size>MAX_BYTES: raise ValueError('Original exceeds byte limit')
|
|
file.write(block)
|
|
storage.put(key,file,size,att['content_type'],att['sha256'])
|
|
finally: obj['Body'].close()
|
|
job_id=uuid4()
|
|
db.execute("UPDATE attachments SET pipeline_version=%s,status='queued',staging_key=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(version,key,attachment_id))
|
|
db.execute('INSERT INTO jobs(id,attachment_id,pipeline_version) VALUES (%s,%s,%s)',(job_id,attachment_id,version))
|
|
db.execute('INSERT INTO outbox(job_id) VALUES (%s)',(job_id,))
|
|
audit(db,job_id,'operator_upgrade',previous_version=att['pipeline_version'],version=version)
|
|
return {'job_id':str(job_id),'pipeline_version':version,'status':'queued'}
|
|
finally: storage.client.close()
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser()
|
|
commands=parser.add_subparsers(dest='command',required=True)
|
|
commands.add_parser('failed')
|
|
commands.add_parser('replay').add_argument('job_id',type=UUID)
|
|
p=commands.add_parser('upgrade'); p.add_argument('attachment_id',type=UUID); p.add_argument('version',type=int)
|
|
args=parser.parse_args()
|
|
if args.command=='failed':
|
|
with database() as db:
|
|
result=db.execute("SELECT id,attachment_id,pipeline_version,attempts,error_code,updated_at FROM jobs WHERE status='failed' ORDER BY updated_at DESC LIMIT 100").fetchall()
|
|
elif args.command=='replay': result=replay(args.job_id)
|
|
else: result=upgrade(args.attachment_id,args.version)
|
|
print(json.dumps(result,default=str,indent=2))
|