137 lines
6.0 KiB
Python
137 lines
6.0 KiB
Python
"""DB outbox relay + durable RabbitMQ consumer; redeliveries are idempotent."""
|
|
import json
|
|
import ssl
|
|
import threading
|
|
import time
|
|
import pika
|
|
from opentelemetry import propagate
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from psycopg.types.json import Jsonb
|
|
from app.common import db, get, config, s3, bucket, thumbnail, now
|
|
from app.telemetry import setup
|
|
|
|
tracer, meter, logger, providers = setup('messenger-worker')
|
|
count = meter.create_counter('worker_jobs', unit='{job}')
|
|
duration = meter.create_histogram('worker_job_duration', unit='s')
|
|
QUEUE = 'attachments'
|
|
|
|
|
|
def connection(identity):
|
|
params = pika.URLParameters(config('BROKER_' + identity + '_URL'))
|
|
params.ssl_options = pika.SSLOptions(ssl.create_default_context(cafile=config('LAB_CA')), 'broker')
|
|
params.heartbeat = 120
|
|
params.blocked_connection_timeout = 10
|
|
params.socket_timeout = 5
|
|
conn = pika.BlockingConnection(params)
|
|
channel = conn.channel()
|
|
if identity == 'PRODUCER':
|
|
channel.confirm_delivery()
|
|
return conn, channel
|
|
|
|
|
|
def relay():
|
|
while True:
|
|
try:
|
|
conn, channel = connection('PRODUCER')
|
|
try:
|
|
while True:
|
|
with db() as database:
|
|
row = database.execute('SELECT id,payload FROM outbox WHERE NOT sent LIMIT 1 FOR UPDATE SKIP LOCKED').fetchone()
|
|
if row:
|
|
channel.basic_publish(exchange='', routing_key=QUEUE, body=json.dumps(row['payload']),
|
|
properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'), mandatory=True)
|
|
database.execute('UPDATE outbox SET sent=true WHERE id=%s', (row['id'],))
|
|
conn.process_data_events(time_limit=0.1)
|
|
if not row:
|
|
time.sleep(0.3)
|
|
finally:
|
|
if conn.is_open:
|
|
conn.close()
|
|
except Exception:
|
|
logger.warning('outbox.retry', extra={'event': 'outbox.retry'})
|
|
time.sleep(2)
|
|
|
|
|
|
def update_attachment(database, attachment):
|
|
database.execute("UPDATE objects SET data=%s WHERE kind='attachment' AND id=%s",
|
|
(Jsonb(attachment), attachment['id']))
|
|
|
|
|
|
def process(job):
|
|
# Lock serializes deliveries, including multiple worker processes.
|
|
# If killed, transaction rolls back; deterministic S3 keys are safe to overwrite.
|
|
with db() as database:
|
|
att = get(database, 'attachment', job['attachment_id'], lock=True)
|
|
if not att or att['status'] in ('ready','failed'):
|
|
return
|
|
att['status'] = 'processing'
|
|
update_attachment(database, att)
|
|
with tracer.start_as_current_span('s3.read_staging', record_exception=False, set_status_on_exception=False):
|
|
raw = s3().get_object(Bucket=bucket(), Key='staging/'+att['id'])['Body'].read()
|
|
with tracer.start_as_current_span('image.thumbnail', record_exception=False, set_status_on_exception=False):
|
|
thumb, meta = thumbnail(raw)
|
|
with tracer.start_as_current_span('s3.publish', record_exception=False, set_status_on_exception=False):
|
|
client = s3()
|
|
for name, data, mime in [('original', raw, att['content_type']), ('thumbnail', thumb, 'image/jpeg')]:
|
|
client.put_object(Bucket=bucket(), Key='final/'+att['id']+'/'+name, Body=data, ContentType=mime)
|
|
att.update(status='ready', variants=[meta], updated_at=now(), error_code=None)
|
|
with tracer.start_as_current_span('db.ready', record_exception=False, set_status_on_exception=False):
|
|
update_attachment(database, att)
|
|
|
|
|
|
def consume(channel, method, properties, body):
|
|
job = json.loads(body)
|
|
start = time.monotonic()
|
|
outcome = 'success'
|
|
with tracer.start_as_current_span('attachment.process', context=propagate.extract(job), kind=SpanKind.CONSUMER,
|
|
record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
process(job)
|
|
except Exception:
|
|
outcome = 'error'
|
|
span.set_status(Status(StatusCode.ERROR))
|
|
if job['attempt'] < int(config('JOB_MAX_ATTEMPTS', '5')):
|
|
job['attempt'] += 1
|
|
# Transactional retry intent is saved before acknowledging this delivery.
|
|
with db() as database:
|
|
database.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload,sent=false',
|
|
(job['job_id'], Jsonb(job)))
|
|
else:
|
|
with db() as database:
|
|
att = get(database, 'attachment', job['attachment_id'], lock=True)
|
|
if att and att['status'] != 'ready':
|
|
att.update(status='failed', error_code='retry_exhausted', updated_at=now())
|
|
update_attachment(database, att)
|
|
time.sleep(min(job['attempt'], 5))
|
|
finally:
|
|
labels = {'job_type': 'attachment.process.v1', 'outcome': outcome}
|
|
count.add(1, labels)
|
|
elapsed = time.monotonic()-start
|
|
duration.record(elapsed, labels)
|
|
span.set_attributes(labels)
|
|
logger.log(40 if outcome=='error' else 20, 'job.completed', extra={
|
|
'event': 'job.completed', 'job_id': job['job_id'], 'attempt': job['attempt'],
|
|
'duration_seconds': elapsed, **labels})
|
|
channel.basic_ack(method.delivery_tag)
|
|
|
|
|
|
def main():
|
|
threading.Thread(target=relay, daemon=True).start()
|
|
while True:
|
|
try:
|
|
conn, channel = connection('CONSUMER')
|
|
try:
|
|
channel.basic_qos(prefetch_count=1)
|
|
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
|
|
channel.start_consuming()
|
|
finally:
|
|
if conn.is_open:
|
|
conn.close()
|
|
except Exception:
|
|
logger.warning('consumer.retry', extra={'event': 'consumer.retry'})
|
|
time.sleep(2)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|