211 lines
8.3 KiB
Python
211 lines
8.3 KiB
Python
"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
from uuid import UUID, uuid4
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.config import settings
|
|
from app.db import SessionLocal, engine
|
|
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
|
|
arm_from_env, broker_channel, crash_if_armed,
|
|
event, parse_envelope, retry_or_fail)
|
|
from app.storage import ObjectMissing, storage
|
|
|
|
logger = logging.getLogger("messenger.worker")
|
|
|
|
|
|
class PermanentFailure(Exception):
|
|
pass
|
|
|
|
|
|
async def claim(job: dict) -> dict | None:
|
|
async with SessionLocal() as db:
|
|
result = await db.execute(text("""
|
|
SELECT * FROM attachments WHERE id=:id FOR UPDATE
|
|
"""), {"id": UUID(job["attachment_id"])})
|
|
row = result.mappings().first()
|
|
if row is None or row["job_id"] != UUID(job["job_id"]) or (
|
|
row["pipeline_version"] != job["pipeline_version"] or
|
|
row["attempt"] != job["attempt"] or row["status"] != "queued"
|
|
):
|
|
await db.rollback()
|
|
return None
|
|
# An early/manual duplicate cannot bypass the durable retry date.
|
|
result = await db.execute(text("""
|
|
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
|
|
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
|
|
updated_at=now()
|
|
WHERE id=:id AND next_attempt_at<=now() RETURNING *
|
|
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS})
|
|
claimed = result.mappings().first()
|
|
if claimed is None:
|
|
await db.rollback()
|
|
return None
|
|
claimed = dict(claimed)
|
|
await event(db, claimed, "processing")
|
|
await db.commit()
|
|
return claimed
|
|
|
|
|
|
async def generate(source: Path, directory: Path) -> list[dict]:
|
|
process = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", "app.image_worker", str(source), str(directory),
|
|
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
try:
|
|
async with asyncio.timeout(IMAGE_TIMEOUT):
|
|
code = await process.wait()
|
|
except TimeoutError:
|
|
raise PermanentFailure("image_timeout") from None
|
|
if code == 21:
|
|
raise PermanentFailure("image_resource_limit")
|
|
if code != 0:
|
|
raise PermanentFailure("invalid_image")
|
|
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
|
finally:
|
|
# Includes cancellation of the entire job. Native image code cannot
|
|
# continue using CPU/RAM after its parent timed out.
|
|
if process.returncode is None:
|
|
process.kill()
|
|
await process.wait()
|
|
|
|
|
|
async def build_objects(row: dict) -> list[dict]:
|
|
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
|
|
directory = Path(temporary)
|
|
source = directory / "source"
|
|
if not row["staging_key"] or row["staging_deleted"]:
|
|
raise PermanentFailure("source_missing")
|
|
try:
|
|
chunks, declared_length = await storage.open_download(row["staging_key"])
|
|
except ObjectMissing:
|
|
raise PermanentFailure("source_missing") from None
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
try:
|
|
with source.open("wb") as output:
|
|
async for chunk in chunks:
|
|
size += len(chunk)
|
|
if size > settings.max_upload_bytes:
|
|
raise PermanentFailure("source_too_large")
|
|
digest.update(chunk)
|
|
output.write(chunk)
|
|
finally:
|
|
await chunks.aclose()
|
|
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
|
raise PermanentFailure("source_checksum_mismatch")
|
|
variants = await generate(source, directory)
|
|
# API never publishes directly to this final prefix. Repeating one
|
|
# v1 job writes the same bytes to the same four keys, never appends.
|
|
with source.open("rb") as body:
|
|
await storage.put(row["object_key"], body, size, row["content_type"])
|
|
prefix = row["object_key"].rsplit("/", 1)[0]
|
|
for variant in variants:
|
|
filename = variant.pop("filename")
|
|
key = f"{prefix}/{filename}"
|
|
with (directory / filename).open("rb") as body:
|
|
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
|
|
variant["object_key"] = key
|
|
await crash_if_armed("crash_after_objects_before_commit")
|
|
return variants
|
|
|
|
|
|
async def finish(row: dict, *, variants=None, code: str | None = None,
|
|
permanent: bool = False) -> bool:
|
|
async with SessionLocal() as db:
|
|
result = await db.execute(text("""
|
|
SELECT * FROM attachments WHERE id=:id AND status='processing'
|
|
AND lease_token=:token AND lease_until>now() FOR UPDATE
|
|
"""), {"id": row["id"], "token": row["lease_token"]})
|
|
current = result.mappings().first()
|
|
if current is None:
|
|
# Lease recovery or explicit operator action already owns a
|
|
# newer attempt. Never overwrite its state with an old worker.
|
|
await db.rollback()
|
|
return False
|
|
if code is not None:
|
|
await retry_or_fail(db, current, code, permanent=permanent)
|
|
else:
|
|
await db.execute(text("""
|
|
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
|
|
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
|
|
WHERE id=:id
|
|
"""), {"id": row["id"], "variants": json.dumps(variants)})
|
|
await event(db, current, "ready")
|
|
await db.commit()
|
|
return True
|
|
|
|
|
|
async def handle(message) -> None:
|
|
try:
|
|
job = parse_envelope(message.body)
|
|
except (ValueError, TypeError, KeyError, AttributeError):
|
|
logger.warning("invalid envelope sent to durable DLQ")
|
|
await message.reject(requeue=False)
|
|
return
|
|
row = await claim(job)
|
|
if row is None:
|
|
# A ready/failed/stale attempt is a durable no-op. A duplicate of
|
|
# active processing is covered by the DB lease recovery, so it too
|
|
# can be acknowledged without running a second concurrent decoder.
|
|
await message.ack()
|
|
return
|
|
try:
|
|
async with asyncio.timeout(TOTAL_TIMEOUT):
|
|
variants = await build_objects(row)
|
|
except PermanentFailure as exc:
|
|
await finish(row, code=str(exc), permanent=True)
|
|
except Exception as exc:
|
|
logger.warning("temporary processing failure attachment=%s type=%s",
|
|
row["id"], type(exc).__name__)
|
|
await finish(row, code="processing_failed")
|
|
else:
|
|
if await finish(row, variants=variants):
|
|
await crash_if_armed("crash_after_commit_before_ack")
|
|
logger.info("ready attachment=%s", row["id"])
|
|
# If finish failed, the exception propagates and the connection closes:
|
|
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
|
|
await message.ack()
|
|
|
|
|
|
async def main() -> None:
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
for name in ("aio_pika", "aiormq"):
|
|
logging.getLogger(name).setLevel(logging.CRITICAL)
|
|
await storage.start()
|
|
connection = None
|
|
try:
|
|
await arm_from_env("WORKER_FAULT")
|
|
while True:
|
|
try:
|
|
connection, _, queue = await broker_channel()
|
|
async with queue.iterator() as deliveries:
|
|
async for message in deliveries:
|
|
await handle(message)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning("worker reconnect: %s", type(exc).__name__)
|
|
if connection:
|
|
await connection.close()
|
|
connection = None
|
|
await asyncio.sleep(1)
|
|
finally:
|
|
if connection:
|
|
await connection.close()
|
|
await storage.close()
|
|
await engine.dispose()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|