Files
mtusi2026lab7/app/worker.py
T
dykhanovaa 4b21efc2cc
Skeleton contracts / contracts (pull_request) Canceled after 0s
lab7 completed
2026-11-20 20:37:41 +03:00

137 lines
6.0 KiB
Python

"""DB outbox relay + durable RabbitMQ consumer; redeliveries are idempotent."""
import json
import ssl
import threading
import time
import pika
from opentelemetry import propagate
from opentelemetry.trace import SpanKind, Status, StatusCode
from psycopg.types.json import Jsonb
from app.common import db, get, config, s3, bucket, thumbnail, now
from app.telemetry import setup
tracer, meter, logger, providers = setup('messenger-worker')
count = meter.create_counter('worker_jobs', unit='{job}')
duration = meter.create_histogram('worker_job_duration', unit='s')
QUEUE = 'attachments'
def connection(identity):
params = pika.URLParameters(config('BROKER_' + identity + '_URL'))
params.ssl_options = pika.SSLOptions(ssl.create_default_context(cafile=config('LAB_CA')), 'broker')
params.heartbeat = 120
params.blocked_connection_timeout = 10
params.socket_timeout = 5
conn = pika.BlockingConnection(params)
channel = conn.channel()
if identity == 'PRODUCER':
channel.confirm_delivery()
return conn, channel
def relay():
while True:
try:
conn, channel = connection('PRODUCER')
try:
while True:
with db() as database:
row = database.execute('SELECT id,payload FROM outbox WHERE NOT sent LIMIT 1 FOR UPDATE SKIP LOCKED').fetchone()
if row:
channel.basic_publish(exchange='', routing_key=QUEUE, body=json.dumps(row['payload']),
properties=pika.BasicProperties(delivery_mode=2, content_type='application/json'), mandatory=True)
database.execute('UPDATE outbox SET sent=true WHERE id=%s', (row['id'],))
conn.process_data_events(time_limit=0.1)
if not row:
time.sleep(0.3)
finally:
if conn.is_open:
conn.close()
except Exception:
logger.warning('outbox.retry', extra={'event': 'outbox.retry'})
time.sleep(2)
def update_attachment(database, attachment):
database.execute("UPDATE objects SET data=%s WHERE kind='attachment' AND id=%s",
(Jsonb(attachment), attachment['id']))
def process(job):
# Lock serializes deliveries, including multiple worker processes.
# If killed, transaction rolls back; deterministic S3 keys are safe to overwrite.
with db() as database:
att = get(database, 'attachment', job['attachment_id'], lock=True)
if not att or att['status'] in ('ready','failed'):
return
att['status'] = 'processing'
update_attachment(database, att)
with tracer.start_as_current_span('s3.read_staging', record_exception=False, set_status_on_exception=False):
raw = s3().get_object(Bucket=bucket(), Key='staging/'+att['id'])['Body'].read()
with tracer.start_as_current_span('image.thumbnail', record_exception=False, set_status_on_exception=False):
thumb, meta = thumbnail(raw)
with tracer.start_as_current_span('s3.publish', record_exception=False, set_status_on_exception=False):
client = s3()
for name, data, mime in [('original', raw, att['content_type']), ('thumbnail', thumb, 'image/jpeg')]:
client.put_object(Bucket=bucket(), Key='final/'+att['id']+'/'+name, Body=data, ContentType=mime)
att.update(status='ready', variants=[meta], updated_at=now(), error_code=None)
with tracer.start_as_current_span('db.ready', record_exception=False, set_status_on_exception=False):
update_attachment(database, att)
def consume(channel, method, properties, body):
job = json.loads(body)
start = time.monotonic()
outcome = 'success'
with tracer.start_as_current_span('attachment.process', context=propagate.extract(job), kind=SpanKind.CONSUMER,
record_exception=False, set_status_on_exception=False) as span:
try:
process(job)
except Exception:
outcome = 'error'
span.set_status(Status(StatusCode.ERROR))
if job['attempt'] < int(config('JOB_MAX_ATTEMPTS', '5')):
job['attempt'] += 1
# Transactional retry intent is saved before acknowledging this delivery.
with db() as database:
database.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload,sent=false',
(job['job_id'], Jsonb(job)))
else:
with db() as database:
att = get(database, 'attachment', job['attachment_id'], lock=True)
if att and att['status'] != 'ready':
att.update(status='failed', error_code='retry_exhausted', updated_at=now())
update_attachment(database, att)
time.sleep(min(job['attempt'], 5))
finally:
labels = {'job_type': 'attachment.process.v1', 'outcome': outcome}
count.add(1, labels)
elapsed = time.monotonic()-start
duration.record(elapsed, labels)
span.set_attributes(labels)
logger.log(40 if outcome=='error' else 20, 'job.completed', extra={
'event': 'job.completed', 'job_id': job['job_id'], 'attempt': job['attempt'],
'duration_seconds': elapsed, **labels})
channel.basic_ack(method.delivery_tag)
def main():
threading.Thread(target=relay, daemon=True).start()
while True:
try:
conn, channel = connection('CONSUMER')
try:
channel.basic_qos(prefetch_count=1)
channel.basic_consume(queue=QUEUE, on_message_callback=consume)
channel.start_consuming()
finally:
if conn.is_open:
conn.close()
except Exception:
logger.warning('consumer.retry', extra={'event': 'consumer.retry'})
time.sleep(2)
if __name__ == '__main__':
main()