Files
mtusi2026lab6/src/worker.py
T

577 lines
15 KiB
Python

import hashlib
import io
import logging
import os
import time
import boto3
import pika
import psycopg
from opentelemetry.trace import SpanKind, Status, StatusCode
from src.telemetry import setup_telemetry
from botocore.config import Config
from PIL import Image, ImageOps
from psycopg.rows import dict_row
from src.queue import (
RABBITMQ_QUEUE,
connect,
declare_queue,
parse_job,
)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger("image-worker")
worker_tracer, worker_meter, worker_logger = setup_telemetry(
"messenger-worker"
)
jobs_processed = worker_meter.create_counter(
name="messenger_worker_jobs_processed",
unit="{job}",
description="Image processing job attempts by outcome",
)
job_duration = worker_meter.create_histogram(
name="messenger_worker_job_duration",
unit="s",
description="Image processing duration in seconds",
)
DATABASE_URL = os.environ["DATABASE_URL"]
S3_BUCKET = os.environ["S3_BUCKET"]
MAX_BYTES = 10 * 1024 * 1024
s3 = boto3.client(
"s3",
endpoint_url=os.environ["S3_ENDPOINT"],
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
region_name="us-east-1",
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=3,
read_timeout=15,
retries={"max_attempts": 1},
),
)
class InvalidImage(Exception):
pass
def make_thumbnail(original):
try:
with Image.open(io.BytesIO(original)) as image:
if image.format not in ("PNG", "JPEG"):
raise InvalidImage("Unsupported image")
if image.width * image.height > 40_000_000:
raise InvalidImage("Too many pixels")
image = ImageOps.exif_transpose(image)
image.thumbnail(
(256, 256),
Image.Resampling.LANCZOS,
)
# JPEG does not support transparency.
rgba = image.convert("RGBA")
background = Image.new(
"RGB",
rgba.size,
(255, 255, 255),
)
background.paste(
rgba,
mask=rgba.getchannel("A"),
)
output = io.BytesIO()
background.save(
output,
format="JPEG",
quality=85,
optimize=True,
)
return (
output.getvalue(),
background.width,
background.height,
)
except (OSError, SyntaxError, EOFError) as exc:
raise InvalidImage("Invalid image") from exc
def get_staging_object(key):
response = s3.get_object(
Bucket=S3_BUCKET,
Key=key,
)
stream = response["Body"]
try:
data = stream.read(MAX_BYTES + 1)
finally:
stream.close()
if not data or len(data) > MAX_BYTES:
raise InvalidImage("Invalid staging size")
return data
def process_job(job_id):
"""
Use a PostgreSQL row lock to prevent concurrent
processing of the same logical job.
The transaction commits before RabbitMQ ACK.
"""
with psycopg.connect(
DATABASE_URL,
row_factory=dict_row,
) as conn:
job = conn.execute(
"""
SELECT j.*, a.staging_key,
a.object_key, a.sha256,
a.size_bytes, a.status AS attachment_status
FROM attachment_jobs j
JOIN attachments a
ON a.id = j.attachment_id
WHERE j.id = %s
FOR UPDATE OF j, a
""",
(job_id,),
).fetchone()
if job is None:
logger.info("Unknown job %s, ignoring", job_id)
return
if job["status"] == "ready":
logger.info("Job %s already completed", job_id)
return
if job["status"] == "failed":
logger.info("Job %s already failed", job_id)
return
if job["next_attempt_at"] is not None:
from datetime import datetime, timezone
if job["next_attempt_at"] > datetime.now(timezone.utc):
logger.info("Job %s is not due yet", job_id)
return
attempts = job["attempts"] + 1
if attempts > job["max_attempts"]:
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'failed',
completed_at = NOW(),
updated_at = NOW()
WHERE id = %s
""",
(job_id,),
)
conn.execute(
"""
UPDATE attachments
SET status = 'failed',
error_code = 'processing_failed',
updated_at = NOW()
WHERE id = %s
""",
(job["attachment_id"],),
)
return
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'processing',
attempts = %s,
started_at = NOW(),
updated_at = NOW()
WHERE id = %s
""",
(attempts, job_id),
)
conn.execute(
"""
UPDATE attachments
SET status = 'processing',
updated_at = NOW()
WHERE id = %s
""",
(job["attachment_id"],),
)
try:
original = get_staging_object(
job["staging_key"]
)
if (
len(original) != job["size_bytes"]
or hashlib.sha256(original).hexdigest()
!= job["sha256"]
):
raise InvalidImage("Staging checksum mismatch")
thumbnail, width, height = make_thumbnail(
original
)
attachment_id = str(job["attachment_id"])
# Stable keys make duplicate delivery safe.
original_key = job["object_key"]
thumb_key = (
f"thumbnails/{attachment_id}.jpg"
)
s3.put_object(
Bucket=S3_BUCKET,
Key=original_key,
Body=original,
ContentType="image/png"
if original[:8] == b"\x89PNG\r\n\x1a\n"
else "image/jpeg",
)
s3.put_object(
Bucket=S3_BUCKET,
Key=thumb_key,
Body=thumbnail,
ContentType="image/jpeg",
)
thumb_sha256 = hashlib.sha256(
thumbnail
).hexdigest()
conn.execute(
"""
INSERT INTO attachment_variants
(
attachment_id,
variant_name,
object_key,
content_type,
size_bytes,
sha256,
width,
height
)
VALUES (
%s, 'thumbnail', %s, 'image/jpeg',
%s, %s, %s, %s
)
ON CONFLICT (
attachment_id, variant_name
)
DO UPDATE SET
object_key = EXCLUDED.object_key,
size_bytes = EXCLUDED.size_bytes,
sha256 = EXCLUDED.sha256,
width = EXCLUDED.width,
height = EXCLUDED.height
""",
(
job["attachment_id"],
thumb_key,
len(thumbnail),
thumb_sha256,
width,
height,
),
)
conn.execute(
"""
UPDATE attachments
SET status = 'ready',
error_code = NULL,
updated_at = NOW()
WHERE id = %s
""",
(job["attachment_id"],),
)
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'ready',
completed_at = NOW(),
updated_at = NOW()
WHERE id = %s
""",
(job_id,),
)
logger.info(
"Job %s ready, thumbnail %sx%s",
job_id,
width,
height,
)
except Exception as exc:
logger.exception(
"Processing failed for job %s: %s",
job_id,
type(exc).__name__,
)
if (
attempts >= job["max_attempts"]
or isinstance(exc, InvalidImage)
):
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'failed',
completed_at = NOW(),
updated_at = NOW()
WHERE id = %s
""",
(job_id,),
)
conn.execute(
"""
UPDATE attachments
SET status = 'failed',
error_code = 'processing_failed',
updated_at = NOW()
WHERE id = %s
""",
(job["attachment_id"],),
)
else:
conn.execute(
"""
UPDATE attachment_jobs
SET status = 'queued',
next_attempt_at =
NOW() + INTERVAL '5 seconds',
published_at = NULL,
updated_at = NOW()
WHERE id = %s
""",
(job_id,),
)
conn.execute(
"""
UPDATE attachments
SET status = 'queued',
updated_at = NOW()
WHERE id = %s
""",
(job["attachment_id"],),
)
def get_job_status(job_id):
with psycopg.connect(DATABASE_URL) as conn:
row = conn.execute(
"""
SELECT status
FROM attachment_jobs
WHERE id = %s
""",
(job_id,),
).fetchone()
return row[0] if row else None
def on_message(channel, method, properties, body):
try:
job_id = parse_job(body)
except (ValueError, TypeError, KeyError):
logger.warning("Invalid queue message")
channel.basic_ack(
delivery_tag=method.delivery_tag
)
return
attributes = {
"messaging.system": "rabbitmq",
"messaging.destination.name": RABBITMQ_QUEUE,
"messaging.operation.name": "process",
"messenger.job_type": "image_thumbnail",
}
# A real consumer span around the actual job processing.
with worker_tracer.start_as_current_span(
"image_job.process",
kind=SpanKind.CONSUMER,
attributes=attributes,
) as span:
started = time.monotonic()
try:
previous_status = get_job_status(job_id)
process_job(job_id)
current_status = get_job_status(job_id)
duration = time.monotonic() - started
if previous_status in ("ready", "failed"):
outcome = "duplicate"
elif current_status == "ready":
outcome = "success"
elif current_status == "failed":
outcome = "failed"
elif current_status == "queued":
outcome = "retry"
else:
outcome = "ignored"
span.set_attribute(
"messenger.job.outcome",
outcome,
)
if outcome == "failed":
span.set_status(
Status(StatusCode.ERROR)
)
# Count actual processing attempts, not
# duplicate deliveries of completed jobs.
if outcome in ("success", "failed", "retry"):
metric_attributes = {
"job_type": "image_thumbnail",
"outcome": outcome,
}
jobs_processed.add(
1,
attributes=metric_attributes,
)
job_duration.record(
duration,
attributes=metric_attributes,
)
worker_logger.info(
"Image job handled",
extra={
"event": "image_job_handled",
"job_type": "image_thumbnail",
"outcome": outcome,
"duration_seconds": round(duration, 4),
},
)
# ACK after process_job has committed the
# PostgreSQL transaction.
channel.basic_ack(
delivery_tag=method.delivery_tag
)
except Exception as exc:
span.record_exception(exc)
span.set_status(Status(StatusCode.ERROR))
worker_logger.error(
"Image job processing interrupted",
extra={
"event": "image_job_interrupted",
"job_type": "image_thumbnail",
"error_type": type(exc).__name__,
},
)
logger.exception(
"Unexpected failure for job %s",
job_id,
)
channel.basic_nack(
delivery_tag=method.delivery_tag,
requeue=True,
)
time.sleep(2)
def main():
while True:
try:
connection = connect()
channel = connection.channel()
declare_queue(channel)
channel.basic_qos(
prefetch_count=1
)
channel.basic_consume(
queue=RABBITMQ_QUEUE,
on_message_callback=on_message,
auto_ack=False,
)
logger.info(
"Worker listening on %s",
RABBITMQ_QUEUE,
)
channel.start_consuming()
except Exception:
logger.exception(
"Worker disconnected, reconnecting"
)
time.sleep(3)
finally:
if (
"connection" in locals()
and connection.is_open
):
connection.close()
if __name__ == "__main__":
main()