forked from mtusi2026/mtusi2026lab6
258 lines
12 KiB
Python
258 lines
12 KiB
Python
"""Durable DB intent -> Valkey Streams -> idempotent S3 effects -> DB -> ack."""
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import sys
|
|
import time
|
|
import uuid
|
|
from multiprocessing import Process
|
|
from datetime import datetime, timezone
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from PIL import Image, ImageOps, UnidentifiedImageError
|
|
import psycopg
|
|
from psycopg.rows import dict_row
|
|
import redis
|
|
import telemetry
|
|
|
|
STREAM = 'attachment-jobs-v1'
|
|
GROUP = 'image-workers-v1'
|
|
MAX_ATTEMPTS = int(os.getenv('JOB_MAX_ATTEMPTS', '5'))
|
|
TIMEOUT = int(os.getenv('JOB_TIMEOUT_SECONDS', '60'))
|
|
Image.MAX_IMAGE_PIXELS = int(os.getenv('MAX_IMAGE_PIXELS', '40000000'))
|
|
|
|
|
|
def secret(name):
|
|
return Path(os.environ[name + '_FILE']).read_text().strip()
|
|
|
|
|
|
def database():
|
|
return telemetry.Database(psycopg.connect(secret('DATABASE_URL'), autocommit=True, row_factory=dict_row,
|
|
connect_timeout=5, options='-c statement_timeout=5000'))
|
|
|
|
|
|
def broker():
|
|
return redis.Redis.from_url(secret('BROKER_URL'), decode_responses=True,
|
|
socket_connect_timeout=5, socket_timeout=5)
|
|
|
|
|
|
def event(row):
|
|
return dict(schema_version=1, job_id=str(row['job_id']), type='attachment.process.v1',
|
|
attachment_id=str(row['attachment_id']), pipeline_version=1,
|
|
attempt=row['attempt'] + 1,
|
|
created_at=row['created_at'].isoformat().replace('+00:00', 'Z'))
|
|
|
|
|
|
def dispatch():
|
|
"""Publish before marking: crash can duplicate, but cannot lose committed intent."""
|
|
telemetry.init("messenger-dispatcher")
|
|
while True:
|
|
try:
|
|
with database() as db:
|
|
r = broker()
|
|
measured = 0
|
|
while True:
|
|
if time.monotonic()-measured >= 1:
|
|
q = db.execute("SELECT count(*) AS depth, extract(epoch from min(j.created_at)) AS oldest FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status IN ('queued','processing')").fetchone()
|
|
telemetry.queue.update(depth=q['depth'],oldest=float(q['oldest']) if q['oldest'] else None,up=1)
|
|
measured = time.monotonic()
|
|
with db.transaction():
|
|
rows = db.execute('''SELECT j.* FROM attachment_jobs j
|
|
JOIN attachments a ON a.id=j.attachment_id
|
|
WHERE j.published_at IS NULL AND a.status='queued'
|
|
ORDER BY j.created_at LIMIT 100 FOR UPDATE OF j SKIP LOCKED''').fetchall()
|
|
for row in rows:
|
|
with telemetry.publishing(row) as carrier:
|
|
r.xadd(STREAM, {'job': json.dumps(event(row)), **carrier})
|
|
db.execute('UPDATE attachment_jobs SET published_at=now() WHERE job_id=%s',
|
|
(row['job_id'],))
|
|
time.sleep(0.25)
|
|
except Exception as exc:
|
|
telemetry.queue['up'] = 0
|
|
print('dispatcher reconnect:', type(exc).__name__, flush=True)
|
|
time.sleep(1)
|
|
|
|
|
|
def thumbnail(data):
|
|
with Image.open(io.BytesIO(data)) as source:
|
|
if source.width * source.height > Image.MAX_IMAGE_PIXELS:
|
|
raise ValueError('pixel limit')
|
|
source.load()
|
|
oriented = ImageOps.exif_transpose(source)
|
|
rgba = oriented.convert('RGBA')
|
|
white = Image.new('RGBA', rgba.size, 'white')
|
|
white.alpha_composite(rgba)
|
|
rgb = white.convert('RGB')
|
|
scale = min(1, 256 / rgb.width, 256 / rgb.height)
|
|
size = (max(1, math.floor(rgb.width * scale)), max(1, math.floor(rgb.height * scale)))
|
|
rgb = rgb.resize(size, Image.Resampling.LANCZOS)
|
|
out = io.BytesIO()
|
|
rgb.save(out, format='JPEG', quality=85, exif=b'')
|
|
return out.getvalue(), size
|
|
|
|
|
|
def alarm_handler(*_):
|
|
raise TimeoutError('job deadline')
|
|
|
|
|
|
def handle(db, s3, r, message_id, fields):
|
|
payload = json.loads(fields['job'])
|
|
job_id = str(uuid.UUID(payload['job_id']))
|
|
attachment_id = str(uuid.UUID(payload['attachment_id']))
|
|
# Session advisory lock survives small transaction boundaries, releases on process death.
|
|
locked = db.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
|
|
(attachment_id,)).fetchone()['ok']
|
|
if not locked:
|
|
return
|
|
try:
|
|
row = db.execute('''SELECT a.*, j.job_id, j.attempt, j.staging_key, j.created_at AS job_created_at
|
|
FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id
|
|
WHERE a.id=%s AND j.job_id=%s AND j.pipeline_version=%s''',
|
|
(attachment_id, job_id, payload['pipeline_version'])).fetchone()
|
|
if row is None:
|
|
raise ValueError('unknown job')
|
|
if row['status'] in ('ready', 'failed'):
|
|
r.xack(STREAM, GROUP, message_id)
|
|
return
|
|
if row['attempt'] >= MAX_ATTEMPTS:
|
|
db.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=%s", (attachment_id,))
|
|
r.xack(STREAM, GROUP, message_id)
|
|
return
|
|
with db.transaction():
|
|
db.execute('UPDATE attachment_jobs SET attempt=attempt+1 WHERE job_id=%s', (job_id,))
|
|
db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (attachment_id,))
|
|
telemetry.event('job.processing')
|
|
outcome = 'success'
|
|
signal.alarm(TIMEOUT)
|
|
try:
|
|
bucket = os.environ['S3_BUCKET']
|
|
response = telemetry.call('s3.get_object', s3.get_object, Bucket=bucket, Key=row['staging_key'])
|
|
try:
|
|
data = response['Body'].read(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')) + 1)
|
|
finally:
|
|
response['Body'].close()
|
|
if len(data) != row['size_bytes'] or hashlib.sha256(data).hexdigest() != row['sha256']:
|
|
raise ValueError('original checksum mismatch')
|
|
thumb, (width, height) = thumbnail(data)
|
|
original_key = 'originals/' + attachment_id
|
|
thumb_key = 'variants/' + attachment_id + '/v1/thumbnail'
|
|
telemetry.call('s3.put_original', s3.put_object, Bucket=bucket, Key=original_key, Body=data, ContentType=row['content_type'])
|
|
telemetry.call('s3.put_thumbnail', s3.put_object, Bucket=bucket, Key=thumb_key, Body=thumb, ContentType='image/jpeg')
|
|
variants = [dict(name='thumbnail', content_type='image/jpeg', size_bytes=len(thumb),
|
|
width=width, height=height, sha256=hashlib.sha256(thumb).hexdigest())]
|
|
db.execute("""UPDATE attachments SET status='ready',object_key=%s,variants=%s::jsonb,
|
|
error_code=NULL,updated_at=now() WHERE id=%s""",
|
|
(original_key, json.dumps(variants), attachment_id))
|
|
if telemetry.ready_duration:
|
|
telemetry.ready_duration.record((datetime.now(timezone.utc)-row['job_created_at']).total_seconds(), {'job_type':'attachment.process.v1'})
|
|
telemetry.event('job.ready')
|
|
except Exception as exc:
|
|
permanent = isinstance(exc, (ValueError, UnidentifiedImageError, Image.DecompressionBombError))
|
|
failed = permanent or row['attempt'] + 1 >= MAX_ATTEMPTS
|
|
with db.transaction():
|
|
db.execute('UPDATE attachments SET status=%s,error_code=%s,updated_at=now() WHERE id=%s',
|
|
('failed' if failed else 'queued',
|
|
'invalid_image' if permanent else 'retry_exhausted' if failed else None,
|
|
attachment_id))
|
|
if not failed:
|
|
carrier = telemetry.carrier()
|
|
db.execute('UPDATE attachment_jobs SET published_at=NULL,traceparent=%s,tracestate=%s WHERE job_id=%s', (carrier.get('traceparent',''),carrier.get('tracestate',''),job_id))
|
|
outcome = 'failed' if failed else 'retry'
|
|
finally:
|
|
signal.alarm(0)
|
|
# Both success and retry intent are committed before ack.
|
|
r.xack(STREAM, GROUP, message_id)
|
|
return outcome
|
|
finally:
|
|
db.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))', (attachment_id,))
|
|
|
|
|
|
def consume():
|
|
telemetry.init()
|
|
signal.signal(signal.SIGALRM, alarm_handler)
|
|
consumer = 'worker-' + str(uuid.uuid4())
|
|
while True:
|
|
try:
|
|
with database() as db:
|
|
r = broker()
|
|
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
|
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
|
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
|
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
|
|
config=Config(connect_timeout=3, read_timeout=10,
|
|
retries={'max_attempts': 1},
|
|
s3={'addressing_style': 'path'}))
|
|
try:
|
|
r.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
|
except redis.ResponseError as exc:
|
|
if 'BUSYGROUP' not in str(exc):
|
|
raise
|
|
while True:
|
|
# Reclaim abandoned deliveries. Advisory lock excludes a still active worker.
|
|
claimed = r.xautoclaim(STREAM, GROUP, consumer, min_idle_time=3000, count=1)[1]
|
|
batches = [(STREAM, claimed)] if claimed else r.xreadgroup(
|
|
GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
|
|
for _, entries in batches:
|
|
for message_id, fields in entries:
|
|
telemetry.run_job(handle, db, s3, r, message_id, fields)
|
|
except Exception as exc:
|
|
print('worker reconnect:', type(exc).__name__, flush=True)
|
|
time.sleep(1)
|
|
|
|
|
|
def cleanup():
|
|
"""Explicit maintenance: only terminal jobs older than the retention interval."""
|
|
retention = int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))
|
|
if retention < 3600:
|
|
raise ValueError('retention must be at least one hour')
|
|
s3 = boto3.client('s3', endpoint_url=os.environ['S3_ENDPOINT'],
|
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
|
aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
|
|
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'))
|
|
with database() as db:
|
|
rows = db.execute("""SELECT j.staging_key FROM attachment_jobs j
|
|
JOIN attachments a ON a.id=j.attachment_id
|
|
WHERE a.status IN ('ready','failed')
|
|
AND a.updated_at < now()-make_interval(secs => %s)""", (retention,)).fetchall()
|
|
for row in rows:
|
|
s3.delete_object(Bucket=os.environ['S3_BUCKET'], Key=row['staging_key'])
|
|
print('terminal staging cleanup:', len(rows), 'eligible keys')
|
|
|
|
|
|
def shutdown(*_):
|
|
raise SystemExit(0)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
signal.signal(signal.SIGTERM, shutdown)
|
|
if MAX_ATTEMPTS < 1 or TIMEOUT < 1:
|
|
raise ValueError('invalid retry configuration')
|
|
if len(sys.argv) > 1 and sys.argv[1] == 'dispatch':
|
|
dispatch()
|
|
elif len(sys.argv) > 1 and sys.argv[1] == 'cleanup':
|
|
cleanup()
|
|
else:
|
|
count = int(os.getenv('WORKER_CONCURRENCY', '1'))
|
|
if count < 1:
|
|
raise ValueError('invalid concurrency')
|
|
if count == 1:
|
|
consume()
|
|
else:
|
|
children = [Process(target=consume) for _ in range(count)]
|
|
for child in children:
|
|
child.start()
|
|
try:
|
|
for child in children:
|
|
child.join()
|
|
finally:
|
|
for child in children:
|
|
if child.is_alive():
|
|
child.terminate()
|
|
for child in children:
|
|
child.join(timeout=10)
|