Files
mtusi2026lab6/worker/main.py
T

258 lines
12 KiB
Python

"""Durable DB intent -> Valkey Streams -> idempotent S3 effects -> DB -> ack."""
import hashlib
import io
import json
import math
import os
from pathlib import Path
import signal
import sys
import time
import uuid
from multiprocessing import Process
from datetime import datetime, timezone
import boto3
from botocore.config import Config
from PIL import Image, ImageOps, UnidentifiedImageError
import psycopg
from psycopg.rows import dict_row
import redis
import telemetry
STREAM = 'attachment-jobs-v1'
GROUP = 'image-workers-v1'
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
TIMEOUT = int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))
Image.MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
def secret(name):
return Path(os.environ[name + '_FILE']).read_text().strip()
def database():
return telemetry.Database(psycopg.connect(secret('DATABASE_URL'), autocommit=True, row_factory=dict_row,
connect_timeout=5, options='-c statement_timeout=5000'))
def broker():
return redis.Redis.from_url(secret('BROKER_URL'), decode_responses=True,
socket_connect_timeout=5, socket_timeout=5)
def event(row):
return dict(schema_version=1, job_id=str(row['job_id']), type='attachment.process.v1',
attachment_id=str(row['attachment_id']), pipeline_version=1,
attempt=row['attempt'] + 1,
created_at=row['created_at'].isoformat().replace('+00:00', 'Z'))
def dispatch():
"""Publish before marking: crash can duplicate, but cannot lose committed intent."""
telemetry.init("messenger-dispatcher")
while True:
try:
with database() as db:
r = broker()
measured = 0
while True:
if time.monotonic()-measured >= 1:
q = db.execute("SELECT count(*) AS depth, extract(epoch from min(j.created_at)) AS oldest FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing')").fetchone()
telemetry.queue.update(depth=q['depth'],oldest=float(q['oldest']) if q['oldest'] else None,up=1)
measured = time.monotonic()
with db.transaction():
rows = db.execute('''SELECT j.* FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id
WHERE j.published_at IS NULL AND a.status='queued'
ORDER BY j.created_at LIMIT 100 FOR UPDATE OF j SKIP LOCKED''').fetchall()
for row in rows:
with telemetry.publishing(row) as carrier:
r.xadd(STREAM, {'job': json.dumps(event(row)), **carrier})
db.execute('UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s',
(row['job_id'],))
time.sleep(0.25)
except Exception as exc:
telemetry.queue['up'] = 0
print('dispatcher reconnect:', type(exc).__name__, flush=True)
time.sleep(1)
def thumbnail(data):
with Image.open(io.BytesIO(data)) as source:
if source.width * source.height > Image.MAX_IMAGE_PIXELS:
raise ValueError('pixel limit')
source.load()
oriented = ImageOps.exif_transpose(source)
rgba = oriented.convert('RGBA')
white = Image.new('RGBA', rgba.size, 'white')
white.alpha_composite(rgba)
rgb = white.convert('RGB')
scale = min(1, 256 / rgb.width, 256 / rgb.height)
size = (max(1, math.floor(rgb.width * scale)), max(1, math.floor(rgb.height * scale)))
rgb = rgb.resize(size, Image.Resampling.LANCZOS)
out = io.BytesIO()
rgb.save(out, format='JPEG', quality=85, exif=b'')
return out.getvalue(), size
def alarm_handler(*_):
raise TimeoutError('job deadline')
def handle(db, s3, r, message_id, fields):
payload = json.loads(fields['job'])
job_id = str(uuid.UUID(payload['job_id']))
attachment_id = str(uuid.UUID(payload['attachment_id']))
# Session advisory lock survives small transaction boundaries, releases on process death.
locked = db.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
(attachment_id,)).fetchone()['ok']
if not locked:
return
try:
row = db.execute('''SELECT a.*, j.job_id, j.attempt, j.staging_key, j.created_at AS job_created_at
FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id
WHERE a.id=%s AND j.job_id=%s AND j.pipeline_version=%s''',
(attachment_id, job_id, payload['pipeline_version'])).fetchone()
if row is None:
raise ValueError('unknown job')
if row['status'] in ('ready', 'failed'):
r.xack(STREAM, GROUP, message_id)
return
if row['attempt'] >= MAX_ATTEMPTS:
db.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s", (attachment_id,))
r.xack(STREAM, GROUP, message_id)
return
with db.transaction():
db.execute('UPDATE attachment_jobs SET attempt=attempt+1 WHERE job_id=%s', (job_id,))
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (attachment_id,))
telemetry.event('job.processing')
outcome = 'success'
signal.alarm(TIMEOUT)
try:
bucket = os.environ['S3_BUCKET']
response = telemetry.call('s3.get_object', s3.get_object, Bucket=bucket, Key=row['staging_key'])
try:
data = response['Body'].read(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')) + 1)
finally:
response['Body'].close()
if len(data) != row['size_bytes'] or hashlib.sha256(data).hexdigest() != row['sha256']:
raise ValueError('original checksum mismatch')
thumb, (width, height) = thumbnail(data)
original_key = 'originals/' + attachment_id
thumb_key = 'variants/' + attachment_id + '/v1/thumbnail'
telemetry.call('s3.put_original', s3.put_object, Bucket=bucket, Key=original_key, Body=data, ContentType=row['content_type'])
telemetry.call('s3.put_thumbnail', s3.put_object, Bucket=bucket, Key=thumb_key, Body=thumb, ContentType='image/jpeg')
variants = [dict(name='thumbnail', content_type='image/jpeg', size_bytes=len(thumb),
width=width, height=height, sha256=hashlib.sha256(thumb).hexdigest())]
db.execute("""UPDATE attachments SET status='ready',object_key=%s,variants=%s::jsonb,
error_code=NULL,updated_at=now() WHERE id=%s""",
(original_key, json.dumps(variants), attachment_id))
if telemetry.ready_duration:
telemetry.ready_duration.record((datetime.now(timezone.utc)-row['job_created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
telemetry.event('job.ready')
except Exception as exc:
permanent = isinstance(exc, (ValueError, UnidentifiedImageError, Image.DecompressionBombError))
failed = permanent or row['attempt'] + 1 >= MAX_ATTEMPTS
with db.transaction():
db.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=now() WHERE id=%s',
('failed' if failed else 'queued',
'invalid_image' if permanent else 'retry_exhausted' if failed else None,
attachment_id))
if not failed:
carrier = telemetry.carrier()
db.execute('UPDATE attachment_jobs SET published_at=NULL,traceparent=%s,tracestate=%s WHERE job_id=%s', (carrier.get('traceparent',''),carrier.get('tracestate',''),job_id))
outcome = 'failed' if failed else 'retry'
finally:
signal.alarm(0)
# Both success and retry intent are committed before ack.
r.xack(STREAM, GROUP, message_id)
return outcome
finally:
db.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))', (attachment_id,))
def consume():
telemetry.init()
signal.signal(signal.SIGALRM, alarm_handler)
consumer = 'worker-' + str(uuid.uuid4())
while True:
try:
with database() as db:
r = broker()
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=3, read_timeout=10,
retries={'max_attempts': 1},
s3={'addressing_style': 'path'}))
try:
r.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
except redis.ResponseError as exc:
if 'BUSYGROUP' not in str(exc):
raise
while True:
# Reclaim abandoned deliveries. Advisory lock excludes a still active worker.
claimed = r.xautoclaim(STREAM, GROUP, consumer, min_idle_time=3000, count=1)[1]
batches = [(STREAM, claimed)] if claimed else r.xreadgroup(
GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
for _, entries in batches:
for message_id, fields in entries:
telemetry.run_job(handle, db, s3, r, message_id, fields)
except Exception as exc:
print('worker reconnect:', type(exc).__name__, flush=True)
time.sleep(1)
def cleanup():
"""Explicit maintenance: only terminal jobs older than the retention interval."""
retention = int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))
if retention < 3600:
raise ValueError('retention must be at least one hour')
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'))
with database() as db:
rows = db.execute("""SELECT j.staging_key FROM attachment_jobs j
JOIN attachments a ON a.id=j.attachment_id
WHERE a.status IN ('ready','failed')
AND a.updated_at < now()-make_interval(secs => %s)""", (retention,)).fetchall()
for row in rows:
s3.delete_object(Bucket=os.environ['S3_BUCKET'], Key=row['staging_key'])
print('terminal staging cleanup:', len(rows), 'eligible keys')
def shutdown(*_):
raise SystemExit(0)
if __name__ == '__main__':
signal.signal(signal.SIGTERM, shutdown)
if MAX_ATTEMPTS < 1 or TIMEOUT < 1:
raise ValueError('invalid retry configuration')
if len(sys.argv) > 1 and sys.argv[1] == 'dispatch':
dispatch()
elif len(sys.argv) > 1 and sys.argv[1] == 'cleanup':
cleanup()
else:
count = int(os.getenv('WORKER_CONCURRENCY', '1'))
if count < 1:
raise ValueError('invalid concurrency')
if count == 1:
consume()
else:
children = [Process(target=consume) for _ in range(count)]
for child in children:
child.start()
try:
for child in children:
child.join()
finally:
for child in children:
if child.is_alive():
child.terminate()
for child in children:
child.join(timeout=10)