forked from mtusi2026/mtusi2026lab6
266 lines
11 KiB
Python
266 lines
11 KiB
Python
"""RabbitMQ consumer for the version-1 attachment pipeline."""
|
|
import asyncio
|
|
from hashlib import sha256
|
|
from io import BytesIO
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import pickle
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from uuid import UUID, uuid4
|
|
|
|
from PIL import Image, ImageOps
|
|
|
|
from .broker import QUEUE, _connection, publish_dlq
|
|
from .config import load_settings
|
|
from .storage.postgres import PostgresStore
|
|
from .storage.s3 import S3Settings, S3Store
|
|
from . import telemetry as tel
|
|
from datetime import datetime, timezone
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s")
|
|
|
|
|
|
class PermanentFailure(Exception):
|
|
pass
|
|
|
|
|
|
def fit(image, bound):
|
|
longest = max(image.size)
|
|
if longest <= bound:
|
|
return image.copy()
|
|
return image.resize((max(1, image.width * bound // longest),
|
|
max(1, image.height * bound // longest)), Image.Resampling.LANCZOS)
|
|
|
|
|
|
def render_isolated(data, timeout):
|
|
try:
|
|
result = subprocess.run([sys.executable, '-m', 'src.render'], input=data,
|
|
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout)
|
|
except subprocess.TimeoutExpired as error:
|
|
raise TimeoutError('render deadline exceeded') from error
|
|
if result.returncode in (20, 21):
|
|
raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit')
|
|
if result.returncode:
|
|
raise RuntimeError('renderer exited')
|
|
# Only our own child process produces this payload; no external pickle input.
|
|
return pickle.loads(result.stdout)
|
|
|
|
|
|
def thumbnail(data: bytes) -> tuple[bytes, int, int]:
|
|
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
|
|
image = ImageOps.exif_transpose(source)
|
|
if image.mode in {"RGBA", "LA"} or (image.mode == "P" and "transparency" in image.info):
|
|
background = Image.new("RGBA", image.size, "white")
|
|
background.alpha_composite(image.convert("RGBA"))
|
|
image = background.convert("RGB")
|
|
else:
|
|
image = image.convert("RGB")
|
|
image = fit(image, 256)
|
|
output = BytesIO()
|
|
image.save(output, format="JPEG", quality=85, optimize=True)
|
|
return output.getvalue(), image.width, image.height
|
|
|
|
|
|
def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]:
|
|
"""Render required variants without preserving EXIF or other input metadata."""
|
|
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
|
|
if source.width * source.height > max_pixels:
|
|
raise ValueError("image exceeds configured pixel limit")
|
|
image = ImageOps.exif_transpose(source).convert("RGBA")
|
|
white = Image.new("RGBA", image.size, "white")
|
|
white.alpha_composite(image)
|
|
rgb = white.convert("RGB")
|
|
|
|
thumb = fit(rgb, 256)
|
|
thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True)
|
|
|
|
side = min(rgb.width, rgb.height)
|
|
left, top = (rgb.width - side) // 2, (rgb.height - side) // 2
|
|
crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
|
|
crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True)
|
|
|
|
optimized = fit(rgb, 1280)
|
|
optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4)
|
|
return [
|
|
("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height),
|
|
("crop", crop_out.getvalue(), "image/jpeg", 128, 128),
|
|
("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height),
|
|
]
|
|
|
|
|
|
async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str:
|
|
# Excludes cleanup for the whole lifetime of S3 effects, including stale attempts.
|
|
async with store.attachment_transaction():
|
|
return await process_locked(store, s3, payload, lease_seconds)
|
|
|
|
|
|
async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str:
|
|
attachment_id = UUID(payload["attachment_id"])
|
|
attachment = await store.claim_attachment_job(attachment_id, lease_seconds)
|
|
if attachment is None:
|
|
return "skipped"
|
|
payload['_claim_time'] = attachment['claim_time']
|
|
payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1))
|
|
attempt_key = uuid4().hex
|
|
original_key = f"originals/{attachment_id}/{attempt_key}"
|
|
await s3.copy(attachment["staging_key"], original_key, attachment["content_type"])
|
|
original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"])
|
|
rendered = await asyncio.to_thread(render_isolated, original, lease_seconds)
|
|
metadata = []
|
|
for name, content, content_type, width, height in rendered:
|
|
extension = "webp" if name == "optimized" else "jpg"
|
|
object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}"
|
|
await s3.put(object_key, BytesIO(content), len(content), content_type)
|
|
metadata.append({"name": name, "object_key": object_key, "content_type": content_type,
|
|
"size_bytes": len(content), "width": width, "height": height,
|
|
"sha256": sha256(content).hexdigest()})
|
|
# Explicit, opt-in integration fault. Marker survives container restart,
|
|
# so one crash exercises lease recovery without a permanent crash loop.
|
|
if os.environ.get('WORKER_FAULT_AFTER_S3_PUT') == '1':
|
|
marker = Path('/tmp/worker-fault-after-s3')
|
|
try:
|
|
with marker.open('x') as stream:
|
|
stream.write(str(attachment_id))
|
|
except FileExistsError:
|
|
pass
|
|
else:
|
|
os._exit(70)
|
|
committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time'])
|
|
if committed:
|
|
try:
|
|
duration = await store.ready_seconds(attachment_id)
|
|
if duration is not None:
|
|
tel.ready_duration.record(duration, {"job_type": "attachment"})
|
|
except Exception:
|
|
tel.log.warning("telemetry.unavailable")
|
|
try:
|
|
await s3.delete(attachment["staging_key"])
|
|
except Exception:
|
|
logging.warning('staging cleanup deferred attachment_id=%s', attachment_id)
|
|
|
|
return "success" if committed else "skipped"
|
|
|
|
def settle_delivery(channel, delivery_tag, future):
|
|
"""Only a successful durable disposition permits acknowledgement."""
|
|
try:
|
|
success = future.result() is True
|
|
except Exception:
|
|
success = False
|
|
if channel.is_open:
|
|
if success:
|
|
channel.basic_ack(delivery_tag)
|
|
else:
|
|
channel.basic_nack(delivery_tag, requeue=True)
|
|
|
|
|
|
def main() -> None:
|
|
tel.setup("messenger-worker")
|
|
settings = load_settings()
|
|
connection = _connection()
|
|
channel = connection.channel()
|
|
workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2")))
|
|
tel.capacity.add(workers)
|
|
channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers)))))
|
|
|
|
async def handle_async(payload):
|
|
"""Each executor thread owns its event loop and its DB/S3 clients."""
|
|
store = PostgresStore(settings.database_url)
|
|
s3 = S3Store(S3Settings.load())
|
|
await store.connect()
|
|
try:
|
|
return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))))
|
|
finally:
|
|
s3.client.close()
|
|
await store.close()
|
|
|
|
async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None):
|
|
retry_store = PostgresStore(settings.database_url)
|
|
await retry_store.connect()
|
|
try:
|
|
return await retry_store.retry_attachment_job(
|
|
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code
|
|
)
|
|
finally:
|
|
await retry_store.close()
|
|
|
|
def handle_job(body):
|
|
payload = None
|
|
try:
|
|
payload = json.loads(body)
|
|
if not isinstance(payload, dict):
|
|
raise ValueError('job must be an object')
|
|
if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1:
|
|
raise ValueError("unexpected broker payload")
|
|
UUID(payload['attachment_id'])
|
|
UUID(payload['job_id'])
|
|
tel.trace.get_current_span().set_attribute("messaging.retry.attempt", int(payload.get('attempt', 1)))
|
|
except (ValueError, KeyError, TypeError):
|
|
try:
|
|
asyncio.run(publish_dlq({'failure': 'invalid_job'}))
|
|
return True, "failed"
|
|
except Exception:
|
|
return False, "failed"
|
|
try:
|
|
outcome = asyncio.run(handle_async(payload))
|
|
except Exception as error:
|
|
logging.warning("job failed attachment_id=%s", payload['attachment_id'])
|
|
if '_claim_time' not in payload:
|
|
return False, "failed"
|
|
try:
|
|
attachment_id = UUID(payload["attachment_id"])
|
|
attempt = int(payload.get("attempt", 1))
|
|
delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5)
|
|
asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'],
|
|
str(error) if isinstance(error, PermanentFailure) else None))
|
|
except Exception:
|
|
return False, "failed"
|
|
return True, "failed"
|
|
return True, outcome
|
|
|
|
def handle(body, headers):
|
|
started = time.monotonic()
|
|
tel.busy.add(1)
|
|
with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
acknowledged, outcome = handle_job(body)
|
|
return acknowledged
|
|
except Exception:
|
|
outcome = "failed"
|
|
raise
|
|
finally:
|
|
if 'outcome' not in locals(): outcome = "failed"
|
|
if outcome == "failed": span.set_status(tel.Status(tel.StatusCode.ERROR))
|
|
attrs = {"job_type": "attachment", "outcome": outcome}
|
|
tel.jobs.add(1, attrs)
|
|
tel.job_duration.record(time.monotonic() - started, attrs)
|
|
tel.log.info("job.processed", extra={"outcome": outcome})
|
|
tel.busy.add(-1)
|
|
|
|
def consume(ch, method, properties, body):
|
|
future = executor.submit(handle, body, properties.headers or {})
|
|
def acknowledged(_future):
|
|
if connection.is_open:
|
|
connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future))
|
|
future.add_done_callback(acknowledged)
|
|
|
|
try:
|
|
def heartbeat():
|
|
Path('/tmp/worker-heartbeat').write_text(str(time.time()))
|
|
connection.call_later(2, heartbeat)
|
|
heartbeat()
|
|
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor:
|
|
channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False)
|
|
channel.start_consuming()
|
|
finally:
|
|
connection.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|