577 lines
15 KiB
Python
577 lines
15 KiB
Python
|
|
import hashlib
|
|
import io
|
|
import logging
|
|
import os
|
|
import time
|
|
|
|
import boto3
|
|
import pika
|
|
import psycopg
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from src.telemetry import setup_telemetry
|
|
from botocore.config import Config
|
|
from PIL import Image, ImageOps
|
|
from psycopg.rows import dict_row
|
|
|
|
from src.queue import (
|
|
RABBITMQ_QUEUE,
|
|
connect,
|
|
declare_queue,
|
|
parse_job,
|
|
)
|
|
|
|
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(message)s",
|
|
)
|
|
|
|
logger = logging.getLogger("image-worker")
|
|
|
|
worker_tracer, worker_meter, worker_logger = setup_telemetry(
|
|
"messenger-worker"
|
|
)
|
|
|
|
jobs_processed = worker_meter.create_counter(
|
|
name="messenger_worker_jobs_processed",
|
|
unit="{job}",
|
|
description="Image processing job attempts by outcome",
|
|
)
|
|
|
|
job_duration = worker_meter.create_histogram(
|
|
name="messenger_worker_job_duration",
|
|
unit="s",
|
|
description="Image processing duration in seconds",
|
|
)
|
|
|
|
DATABASE_URL = os.environ["DATABASE_URL"]
|
|
S3_BUCKET = os.environ["S3_BUCKET"]
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
|
|
s3 = boto3.client(
|
|
"s3",
|
|
endpoint_url=os.environ["S3_ENDPOINT"],
|
|
aws_access_key_id=os.environ["S3_ACCESS_KEY"],
|
|
aws_secret_access_key=os.environ["S3_SECRET_KEY"],
|
|
region_name="us-east-1",
|
|
config=Config(
|
|
signature_version="s3v4",
|
|
s3={"addressing_style": "path"},
|
|
connect_timeout=3,
|
|
read_timeout=15,
|
|
retries={"max_attempts": 1},
|
|
),
|
|
)
|
|
|
|
|
|
class InvalidImage(Exception):
|
|
pass
|
|
|
|
|
|
def make_thumbnail(original):
|
|
try:
|
|
with Image.open(io.BytesIO(original)) as image:
|
|
if image.format not in ("PNG", "JPEG"):
|
|
raise InvalidImage("Unsupported image")
|
|
|
|
if image.width * image.height > 40_000_000:
|
|
raise InvalidImage("Too many pixels")
|
|
|
|
image = ImageOps.exif_transpose(image)
|
|
image.thumbnail(
|
|
(256, 256),
|
|
Image.Resampling.LANCZOS,
|
|
)
|
|
|
|
# JPEG does not support transparency.
|
|
rgba = image.convert("RGBA")
|
|
|
|
background = Image.new(
|
|
"RGB",
|
|
rgba.size,
|
|
(255, 255, 255),
|
|
)
|
|
|
|
background.paste(
|
|
rgba,
|
|
mask=rgba.getchannel("A"),
|
|
)
|
|
|
|
output = io.BytesIO()
|
|
|
|
background.save(
|
|
output,
|
|
format="JPEG",
|
|
quality=85,
|
|
optimize=True,
|
|
)
|
|
|
|
return (
|
|
output.getvalue(),
|
|
background.width,
|
|
background.height,
|
|
)
|
|
|
|
except (OSError, SyntaxError, EOFError) as exc:
|
|
raise InvalidImage("Invalid image") from exc
|
|
|
|
|
|
def get_staging_object(key):
|
|
response = s3.get_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=key,
|
|
)
|
|
|
|
stream = response["Body"]
|
|
|
|
try:
|
|
data = stream.read(MAX_BYTES + 1)
|
|
finally:
|
|
stream.close()
|
|
|
|
if not data or len(data) > MAX_BYTES:
|
|
raise InvalidImage("Invalid staging size")
|
|
|
|
return data
|
|
|
|
|
|
def process_job(job_id):
|
|
"""
|
|
Use a PostgreSQL row lock to prevent concurrent
|
|
processing of the same logical job.
|
|
|
|
The transaction commits before RabbitMQ ACK.
|
|
"""
|
|
|
|
with psycopg.connect(
|
|
DATABASE_URL,
|
|
row_factory=dict_row,
|
|
) as conn:
|
|
|
|
job = conn.execute(
|
|
"""
|
|
SELECT j.*, a.staging_key,
|
|
a.object_key, a.sha256,
|
|
a.size_bytes, a.status AS attachment_status
|
|
FROM attachment_jobs j
|
|
JOIN attachments a
|
|
ON a.id = j.attachment_id
|
|
WHERE j.id = %s
|
|
FOR UPDATE OF j, a
|
|
""",
|
|
(job_id,),
|
|
).fetchone()
|
|
|
|
if job is None:
|
|
logger.info("Unknown job %s, ignoring", job_id)
|
|
return
|
|
|
|
if job["status"] == "ready":
|
|
logger.info("Job %s already completed", job_id)
|
|
return
|
|
|
|
if job["status"] == "failed":
|
|
logger.info("Job %s already failed", job_id)
|
|
return
|
|
|
|
if job["next_attempt_at"] is not None:
|
|
from datetime import datetime, timezone
|
|
|
|
if job["next_attempt_at"] > datetime.now(timezone.utc):
|
|
logger.info("Job %s is not due yet", job_id)
|
|
return
|
|
|
|
attempts = job["attempts"] + 1
|
|
|
|
if attempts > job["max_attempts"]:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'failed',
|
|
completed_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job_id,),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed',
|
|
error_code = 'processing_failed',
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job["attachment_id"],),
|
|
)
|
|
|
|
return
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'processing',
|
|
attempts = %s,
|
|
started_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(attempts, job_id),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'processing',
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job["attachment_id"],),
|
|
)
|
|
|
|
try:
|
|
original = get_staging_object(
|
|
job["staging_key"]
|
|
)
|
|
|
|
if (
|
|
len(original) != job["size_bytes"]
|
|
or hashlib.sha256(original).hexdigest()
|
|
!= job["sha256"]
|
|
):
|
|
raise InvalidImage("Staging checksum mismatch")
|
|
|
|
thumbnail, width, height = make_thumbnail(
|
|
original
|
|
)
|
|
|
|
attachment_id = str(job["attachment_id"])
|
|
|
|
# Stable keys make duplicate delivery safe.
|
|
original_key = job["object_key"]
|
|
thumb_key = (
|
|
f"thumbnails/{attachment_id}.jpg"
|
|
)
|
|
|
|
s3.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=original_key,
|
|
Body=original,
|
|
ContentType="image/png"
|
|
if original[:8] == b"\x89PNG\r\n\x1a\n"
|
|
else "image/jpeg",
|
|
)
|
|
|
|
s3.put_object(
|
|
Bucket=S3_BUCKET,
|
|
Key=thumb_key,
|
|
Body=thumbnail,
|
|
ContentType="image/jpeg",
|
|
)
|
|
|
|
thumb_sha256 = hashlib.sha256(
|
|
thumbnail
|
|
).hexdigest()
|
|
|
|
conn.execute(
|
|
"""
|
|
INSERT INTO attachment_variants
|
|
(
|
|
attachment_id,
|
|
variant_name,
|
|
object_key,
|
|
content_type,
|
|
size_bytes,
|
|
sha256,
|
|
width,
|
|
height
|
|
)
|
|
VALUES (
|
|
%s, 'thumbnail', %s, 'image/jpeg',
|
|
%s, %s, %s, %s
|
|
)
|
|
ON CONFLICT (
|
|
attachment_id, variant_name
|
|
)
|
|
DO UPDATE SET
|
|
object_key = EXCLUDED.object_key,
|
|
size_bytes = EXCLUDED.size_bytes,
|
|
sha256 = EXCLUDED.sha256,
|
|
width = EXCLUDED.width,
|
|
height = EXCLUDED.height
|
|
""",
|
|
(
|
|
job["attachment_id"],
|
|
thumb_key,
|
|
len(thumbnail),
|
|
thumb_sha256,
|
|
width,
|
|
height,
|
|
),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'ready',
|
|
error_code = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job["attachment_id"],),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'ready',
|
|
completed_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job_id,),
|
|
)
|
|
|
|
logger.info(
|
|
"Job %s ready, thumbnail %sx%s",
|
|
job_id,
|
|
width,
|
|
height,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.exception(
|
|
"Processing failed for job %s: %s",
|
|
job_id,
|
|
type(exc).__name__,
|
|
)
|
|
|
|
if (
|
|
attempts >= job["max_attempts"]
|
|
or isinstance(exc, InvalidImage)
|
|
):
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'failed',
|
|
completed_at = NOW(),
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job_id,),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'failed',
|
|
error_code = 'processing_failed',
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job["attachment_id"],),
|
|
)
|
|
|
|
else:
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachment_jobs
|
|
SET status = 'queued',
|
|
next_attempt_at =
|
|
NOW() + INTERVAL '5 seconds',
|
|
published_at = NULL,
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job_id,),
|
|
)
|
|
|
|
conn.execute(
|
|
"""
|
|
UPDATE attachments
|
|
SET status = 'queued',
|
|
updated_at = NOW()
|
|
WHERE id = %s
|
|
""",
|
|
(job["attachment_id"],),
|
|
)
|
|
|
|
|
|
def get_job_status(job_id):
|
|
with psycopg.connect(DATABASE_URL) as conn:
|
|
row = conn.execute(
|
|
"""
|
|
SELECT status
|
|
FROM attachment_jobs
|
|
WHERE id = %s
|
|
""",
|
|
(job_id,),
|
|
).fetchone()
|
|
|
|
return row[0] if row else None
|
|
|
|
|
|
def on_message(channel, method, properties, body):
|
|
try:
|
|
job_id = parse_job(body)
|
|
except (ValueError, TypeError, KeyError):
|
|
logger.warning("Invalid queue message")
|
|
channel.basic_ack(
|
|
delivery_tag=method.delivery_tag
|
|
)
|
|
return
|
|
|
|
attributes = {
|
|
"messaging.system": "rabbitmq",
|
|
"messaging.destination.name": RABBITMQ_QUEUE,
|
|
"messaging.operation.name": "process",
|
|
"messenger.job_type": "image_thumbnail",
|
|
}
|
|
|
|
# A real consumer span around the actual job processing.
|
|
with worker_tracer.start_as_current_span(
|
|
"image_job.process",
|
|
kind=SpanKind.CONSUMER,
|
|
attributes=attributes,
|
|
) as span:
|
|
|
|
started = time.monotonic()
|
|
|
|
try:
|
|
previous_status = get_job_status(job_id)
|
|
|
|
process_job(job_id)
|
|
|
|
current_status = get_job_status(job_id)
|
|
duration = time.monotonic() - started
|
|
|
|
if previous_status in ("ready", "failed"):
|
|
outcome = "duplicate"
|
|
|
|
elif current_status == "ready":
|
|
outcome = "success"
|
|
|
|
elif current_status == "failed":
|
|
outcome = "failed"
|
|
|
|
elif current_status == "queued":
|
|
outcome = "retry"
|
|
|
|
else:
|
|
outcome = "ignored"
|
|
|
|
span.set_attribute(
|
|
"messenger.job.outcome",
|
|
outcome,
|
|
)
|
|
|
|
if outcome == "failed":
|
|
span.set_status(
|
|
Status(StatusCode.ERROR)
|
|
)
|
|
|
|
# Count actual processing attempts, not
|
|
# duplicate deliveries of completed jobs.
|
|
if outcome in ("success", "failed", "retry"):
|
|
metric_attributes = {
|
|
"job_type": "image_thumbnail",
|
|
"outcome": outcome,
|
|
}
|
|
|
|
jobs_processed.add(
|
|
1,
|
|
attributes=metric_attributes,
|
|
)
|
|
|
|
job_duration.record(
|
|
duration,
|
|
attributes=metric_attributes,
|
|
)
|
|
|
|
worker_logger.info(
|
|
"Image job handled",
|
|
extra={
|
|
"event": "image_job_handled",
|
|
"job_type": "image_thumbnail",
|
|
"outcome": outcome,
|
|
"duration_seconds": round(duration, 4),
|
|
},
|
|
)
|
|
|
|
# ACK after process_job has committed the
|
|
# PostgreSQL transaction.
|
|
channel.basic_ack(
|
|
delivery_tag=method.delivery_tag
|
|
)
|
|
|
|
except Exception as exc:
|
|
span.record_exception(exc)
|
|
span.set_status(Status(StatusCode.ERROR))
|
|
|
|
worker_logger.error(
|
|
"Image job processing interrupted",
|
|
extra={
|
|
"event": "image_job_interrupted",
|
|
"job_type": "image_thumbnail",
|
|
"error_type": type(exc).__name__,
|
|
},
|
|
)
|
|
|
|
logger.exception(
|
|
"Unexpected failure for job %s",
|
|
job_id,
|
|
)
|
|
|
|
channel.basic_nack(
|
|
delivery_tag=method.delivery_tag,
|
|
requeue=True,
|
|
)
|
|
|
|
time.sleep(2)
|
|
|
|
|
|
|
|
def main():
|
|
while True:
|
|
try:
|
|
connection = connect()
|
|
channel = connection.channel()
|
|
|
|
declare_queue(channel)
|
|
|
|
channel.basic_qos(
|
|
prefetch_count=1
|
|
)
|
|
|
|
channel.basic_consume(
|
|
queue=RABBITMQ_QUEUE,
|
|
on_message_callback=on_message,
|
|
auto_ack=False,
|
|
)
|
|
|
|
logger.info(
|
|
"Worker listening on %s",
|
|
RABBITMQ_QUEUE,
|
|
)
|
|
|
|
channel.start_consuming()
|
|
|
|
except Exception:
|
|
logger.exception(
|
|
"Worker disconnected, reconnecting"
|
|
)
|
|
time.sleep(3)
|
|
|
|
finally:
|
|
if (
|
|
"connection" in locals()
|
|
and connection.is_open
|
|
):
|
|
connection.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|