Files
mtusi2026lab6/app/worker.py
T
2026-11-20 17:52:56 +03:00

131 lines
5.7 KiB
Python

"""DB outbox relay + durable RabbitMQ consumer; redeliveries are idempotent."""
import json
import threading
import time
import pika
from opentelemetry import propagate
from opentelemetry.trace import SpanKind, Status, StatusCode
from psycopg.types.json import Jsonb
from app.common import db, migrate, get, put, config, s3, bucket, thumbnail, now
from app.telemetry import setup
tracer, meter, logger, providers = setup('messenger-worker')
count = meter.create_counter('worker_jobs', unit='{job}')
duration = meter.create_histogram('worker_job_duration', unit='s')
QUEUE = 'attachments'
def connection():
params = pika.URLParameters(config('BROKER_URL'))
params.heartbeat = 120
params.blocked_connection_timeout = 10
params.socket_timeout = 5
conn = pika.BlockingConnection(params)
channel = conn.channel()
channel.queue_declare(queue=QUEUE, durable=True)
channel.confirm_delivery()
return conn, channel
def relay():
while True:
try:
conn, channel = connection()
try:
while True:
with db() as database:
row = database.execute('SELECT id,payload FROM outbox WHERE NOT sent LIMIT 1 FOR UPDATE SKIP LOCKED').fetchone()
if row:
channel.basic_publish(exchange='', routing_key=QUEUE, body=json.dumps(row['payload']),
properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'), mandatory=True)
database.execute('UPDATE outbox SET sent=true WHERE id=%s', (row['id'],))
conn.process_data_events(time_limit=0.1)
if not row:
time.sleep(0.3)
finally:
if conn.is_open:
conn.close()
except Exception:
logger.warning('outbox.retry', extra={'event': 'outbox.retry'})
time.sleep(2)
def process(job):
# Lock serializes deliveries, including multiple worker processes.
# If killed, transaction rolls back; deterministic S3 keys are safe to overwrite.
with db() as database:
att = get(database, 'attachment', job['attachment_id'], lock=True)
if not att or att['status'] in ('ready','failed'):
return
att['status'] = 'processing'
put(database, 'attachment', att)
with tracer.start_as_current_span('s3.read_staging', record_exception=False, set_status_on_exception=False):
raw = s3().get_object(Bucket=bucket(), Key='staging/'+att['id'])['Body'].read()
with tracer.start_as_current_span('image.thumbnail', record_exception=False, set_status_on_exception=False):
thumb, meta = thumbnail(raw)
with tracer.start_as_current_span('s3.publish', record_exception=False, set_status_on_exception=False):
client = s3()
for name, data, mime in [('original', raw, att['content_type']), ('thumbnail', thumb, 'image/jpeg')]:
client.put_object(Bucket=bucket(), Key='final/'+att['id']+'/'+name, Body=data, ContentType=mime)
att.update(status='ready', variants=[meta], updated_at=now(), error_code=None)
with tracer.start_as_current_span('db.ready', record_exception=False, set_status_on_exception=False):
put(database, 'attachment', att)
def consume(channel, method, properties, body):
job = json.loads(body)
start = time.monotonic()
outcome = 'success'
with tracer.start_as_current_span('attachment.process', context=propagate.extract(job), kind=SpanKind.CONSUMER,
record_exception=False, set_status_on_exception=False) as span:
try:
process(job)
except Exception:
outcome = 'error'
span.set_status(Status(StatusCode.ERROR))
if job['attempt'] < int(config('JOB_MAX_ATTEMPTS', '5')):
job['attempt'] += 1
# Transactional retry intent is saved before acknowledging this delivery.
with db() as database:
database.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload,sent=false',
(job['job_id'], Jsonb(job)))
else:
with db() as database:
att = get(database, 'attachment', job['attachment_id'], lock=True)
if att and att['status'] != 'ready':
att.update(status='failed', error_code='retry_exhausted', updated_at=now())
put(database, 'attachment', att)
time.sleep(min(job['attempt'], 5))
finally:
labels = {'job_type': 'attachment.process.v1', 'outcome': outcome}
count.add(1, labels)
elapsed = time.monotonic()-start
duration.record(elapsed, labels)
span.set_attributes(labels)
logger.log(40 if outcome=='error' else 20, 'job.completed', extra={
'event': 'job.completed', 'job_id': job['job_id'], 'attempt': job['attempt'],
'duration_seconds': elapsed, **labels})
channel.basic_ack(method.delivery_tag)
def main():
migrate()
threading.Thread(target=relay, daemon=True).start()
while True:
try:
conn, channel = connection()
try:
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
channel.start_consuming()
finally:
if conn.is_open:
conn.close()
except Exception:
logger.warning('consumer.retry', extra={'event': 'consumer.retry'})
time.sleep(2)
if __name__ == '__main__':
main()