Files
mtusi2026lab6/src/worker.py
T
2026-11-06 11:12:38 +03:00

266 lines
11 KiB
Python

"""RabbitMQ consumer for the version-1 attachment pipeline."""
import asyncio
from hashlib import sha256
from io import BytesIO
import json
import logging
import os
import random
import pickle
import subprocess
import sys
import time
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor
from uuid import UUID, uuid4
from PIL import Image, ImageOps
from .broker import QUEUE, _connection, publish_dlq
from .config import load_settings
from .storage.postgres import PostgresStore
from .storage.s3 import S3Settings, S3Store
from . import telemetry as tel
from datetime import datetime, timezone
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s worker %(message)s")
class PermanentFailure(Exception):
pass
def fit(image, bound):
longest = max(image.size)
if longest <= bound:
return image.copy()
return image.resize((max(1, image.width * bound // longest),
max(1, image.height * bound // longest)), Image.Resampling.LANCZOS)
def render_isolated(data, timeout):
try:
result = subprocess.run([sys.executable, '-m', 'src.render'], input=data,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout)
except subprocess.TimeoutExpired as error:
raise TimeoutError('render deadline exceeded') from error
if result.returncode in (20, 21):
raise PermanentFailure('invalid_image' if result.returncode == 20 else 'image_resource_limit')
if result.returncode:
raise RuntimeError('renderer exited')
# Only our own child process produces this payload; no external pickle input.
return pickle.loads(result.stdout)
def thumbnail(data: bytes) -> tuple[bytes, int, int]:
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
image = ImageOps.exif_transpose(source)
if image.mode in {"RGBA", "LA"} or (image.mode == "P" and "transparency" in image.info):
background = Image.new("RGBA", image.size, "white")
background.alpha_composite(image.convert("RGBA"))
image = background.convert("RGB")
else:
image = image.convert("RGB")
image = fit(image, 256)
output = BytesIO()
image.save(output, format="JPEG", quality=85, optimize=True)
return output.getvalue(), image.width, image.height
def variants(data: bytes, max_pixels: int = 40_000_000) -> list[tuple[str, bytes, str, int, int]]:
"""Render required variants without preserving EXIF or other input metadata."""
with Image.open(BytesIO(data), formats=("PNG", "JPEG")) as source:
if source.width * source.height > max_pixels:
raise ValueError("image exceeds configured pixel limit")
image = ImageOps.exif_transpose(source).convert("RGBA")
white = Image.new("RGBA", image.size, "white")
white.alpha_composite(image)
rgb = white.convert("RGB")
thumb = fit(rgb, 256)
thumbnail_out = BytesIO(); thumb.save(thumbnail_out, "JPEG", quality=85, optimize=True)
side = min(rgb.width, rgb.height)
left, top = (rgb.width - side) // 2, (rgb.height - side) // 2
crop = rgb.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS)
crop_out = BytesIO(); crop.save(crop_out, "JPEG", quality=85, optimize=True)
optimized = fit(rgb, 1280)
optimized_out = BytesIO(); optimized.save(optimized_out, "WEBP", quality=80, method=4)
return [
("thumbnail", thumbnail_out.getvalue(), "image/jpeg", thumb.width, thumb.height),
("crop", crop_out.getvalue(), "image/jpeg", 128, 128),
("optimized", optimized_out.getvalue(), "image/webp", optimized.width, optimized.height),
]
async def process(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str:
# Excludes cleanup for the whole lifetime of S3 effects, including stale attempts.
async with store.attachment_transaction():
return await process_locked(store, s3, payload, lease_seconds)
async def process_locked(store: PostgresStore, s3: S3Store, payload: dict, lease_seconds: int) -> str:
attachment_id = UUID(payload["attachment_id"])
attachment = await store.claim_attachment_job(attachment_id, lease_seconds)
if attachment is None:
return "skipped"
payload['_claim_time'] = attachment['claim_time']
payload['attempt'] = attachment.get('attempt', payload.get('attempt', 1))
attempt_key = uuid4().hex
original_key = f"originals/{attachment_id}/{attempt_key}"
await s3.copy(attachment["staging_key"], original_key, attachment["content_type"])
original = await s3.download_key(attachment["staging_key"], attachment["size_bytes"], attachment["sha256"])
rendered = await asyncio.to_thread(render_isolated, original, lease_seconds)
metadata = []
for name, content, content_type, width, height in rendered:
extension = "webp" if name == "optimized" else "jpg"
object_key = f"variants/{attachment_id}/{attempt_key}/{name}.{extension}"
await s3.put(object_key, BytesIO(content), len(content), content_type)
metadata.append({"name": name, "object_key": object_key, "content_type": content_type,
"size_bytes": len(content), "width": width, "height": height,
"sha256": sha256(content).hexdigest()})
# Explicit, opt-in integration fault. Marker survives container restart,
# so one crash exercises lease recovery without a permanent crash loop.
if os.environ.get('WORKER_FAULT_AFTER_S3_PUT') == '1':
marker = Path('/tmp/worker-fault-after-s3')
try:
with marker.open('x') as stream:
stream.write(str(attachment_id))
except FileExistsError:
pass
else:
os._exit(70)
committed = await store.complete_attachment_job(attachment_id, original_key, metadata, attachment['claim_time'])
if committed:
try:
duration = await store.ready_seconds(attachment_id)
if duration is not None:
tel.ready_duration.record(duration, {"job_type": "attachment"})
except Exception:
tel.log.warning("telemetry.unavailable")
try:
await s3.delete(attachment["staging_key"])
except Exception:
logging.warning('staging cleanup deferred attachment_id=%s', attachment_id)
return "success" if committed else "skipped"
def settle_delivery(channel, delivery_tag, future):
"""Only a successful durable disposition permits acknowledgement."""
try:
success = future.result() is True
except Exception:
success = False
if channel.is_open:
if success:
channel.basic_ack(delivery_tag)
else:
channel.basic_nack(delivery_tag, requeue=True)
def main() -> None:
tel.setup("messenger-worker")
settings = load_settings()
connection = _connection()
channel = connection.channel()
workers = max(1, int(os.environ.get("WORKER_CONCURRENCY", "2")))
tel.capacity.add(workers)
channel.basic_qos(prefetch_count=max(1, int(os.environ.get("WORKER_PREFETCH", str(workers)))))
async def handle_async(payload):
"""Each executor thread owns its event loop and its DB/S3 clients."""
store = PostgresStore(settings.database_url)
s3 = S3Store(S3Settings.load())
await store.connect()
try:
return await process(store, s3, payload, max(1, int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))))
finally:
s3.client.close()
await store.close()
async def schedule_retry(attachment_id, delay, claim_time, permanent_code=None):
retry_store = PostgresStore(settings.database_url)
await retry_store.connect()
try:
return await retry_store.retry_attachment_job(
attachment_id, max(1, int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))), delay, claim_time, permanent_code
)
finally:
await retry_store.close()
def handle_job(body):
payload = None
try:
payload = json.loads(body)
if not isinstance(payload, dict):
raise ValueError('job must be an object')
if payload["type"] != "attachment.process.v1" or payload["schema_version"] != 1:
raise ValueError("unexpected broker payload")
UUID(payload['attachment_id'])
UUID(payload['job_id'])
tel.trace.get_current_span().set_attribute("messaging.retry.attempt", int(payload.get('attempt', 1)))
except (ValueError, KeyError, TypeError):
try:
asyncio.run(publish_dlq({'failure': 'invalid_job'}))
return True, "failed"
except Exception:
return False, "failed"
try:
outcome = asyncio.run(handle_async(payload))
except Exception as error:
logging.warning("job failed attachment_id=%s", payload['attachment_id'])
if '_claim_time' not in payload:
return False, "failed"
try:
attachment_id = UUID(payload["attachment_id"])
attempt = int(payload.get("attempt", 1))
delay = min(60.0, float(2 ** min(attempt, 6))) * random.uniform(0.5, 1.5)
asyncio.run(schedule_retry(attachment_id, delay, payload['_claim_time'],
str(error) if isinstance(error, PermanentFailure) else None))
except Exception:
return False, "failed"
return True, "failed"
return True, outcome
def handle(body, headers):
started = time.monotonic()
tel.busy.add(1)
with tel.tracer.start_as_current_span("attachment.consume", context=tel.extract(headers), kind=tel.SpanKind.CONSUMER, record_exception=False, set_status_on_exception=False) as span:
try:
acknowledged, outcome = handle_job(body)
return acknowledged
except Exception:
outcome = "failed"
raise
finally:
if 'outcome' not in locals(): outcome = "failed"
if outcome == "failed": span.set_status(tel.Status(tel.StatusCode.ERROR))
attrs = {"job_type": "attachment", "outcome": outcome}
tel.jobs.add(1, attrs)
tel.job_duration.record(time.monotonic() - started, attrs)
tel.log.info("job.processed", extra={"outcome": outcome})
tel.busy.add(-1)
def consume(ch, method, properties, body):
future = executor.submit(handle, body, properties.headers or {})
def acknowledged(_future):
if connection.is_open:
connection.add_callback_threadsafe(lambda: settle_delivery(ch, method.delivery_tag, _future))
future.add_done_callback(acknowledged)
try:
def heartbeat():
Path('/tmp/worker-heartbeat').write_text(str(time.time()))
connection.call_later(2, heartbeat)
heartbeat()
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="attachment-worker") as executor:
channel.basic_consume(queue=QUEUE, on_message_callback=consume, auto_ack=False)
channel.start_consuming()
finally:
connection.close()
if __name__ == "__main__":
main()