258 lines
10 KiB
Python
258 lines
10 KiB
Python
"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
from pathlib import Path
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from uuid import UUID, uuid4
|
|
|
|
from sqlalchemy import text
|
|
|
|
from app.config import settings
|
|
from app.db import SessionLocal, engine
|
|
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
|
|
arm_from_env, broker_channel, crash_if_armed,
|
|
event, parse_envelope, retry_or_fail)
|
|
from app.storage import ObjectMissing, storage
|
|
from app.telemetry import (configure, emit, install_termination_handler, jobs_duration,
|
|
jobs_processed, shutdown, span)
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
|
|
logger = logging.getLogger("messenger.worker")
|
|
|
|
|
|
class PermanentFailure(Exception):
|
|
pass
|
|
|
|
|
|
async def claim(job: dict) -> dict | None:
|
|
async with SessionLocal() as db:
|
|
result = await db.execute(text("""
|
|
SELECT * FROM attachments WHERE id=:id FOR UPDATE
|
|
"""), {"id": UUID(job["attachment_id"])})
|
|
row = result.mappings().first()
|
|
if row is None or row["job_id"] != UUID(job["job_id"]) or (
|
|
row["pipeline_version"] != job["pipeline_version"] or
|
|
row["attempt"] != job["attempt"] or row["status"] != "queued"
|
|
):
|
|
await db.rollback()
|
|
return None
|
|
# An early/manual duplicate cannot bypass the durable retry date.
|
|
result = await db.execute(text("""
|
|
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
|
|
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
|
|
updated_at=now()
|
|
WHERE id=:id AND next_attempt_at<=now() RETURNING *
|
|
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS})
|
|
claimed = result.mappings().first()
|
|
if claimed is None:
|
|
await db.rollback()
|
|
return None
|
|
claimed = dict(claimed)
|
|
await event(db, claimed, "processing")
|
|
await db.commit()
|
|
return claimed
|
|
|
|
|
|
async def generate(source: Path, directory: Path) -> list[dict]:
|
|
process = await asyncio.create_subprocess_exec(
|
|
sys.executable, "-m", "app.image_worker", str(source), str(directory),
|
|
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
|
)
|
|
try:
|
|
try:
|
|
async with asyncio.timeout(IMAGE_TIMEOUT):
|
|
code = await process.wait()
|
|
except TimeoutError:
|
|
raise PermanentFailure("image_timeout") from None
|
|
if code == 21:
|
|
raise PermanentFailure("image_resource_limit")
|
|
if code != 0:
|
|
raise PermanentFailure("invalid_image")
|
|
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
|
finally:
|
|
# Includes cancellation of the entire job. Native image code cannot
|
|
# continue using CPU/RAM after its parent timed out.
|
|
if process.returncode is None:
|
|
process.kill()
|
|
await process.wait()
|
|
|
|
|
|
async def build_objects(row: dict) -> list[dict]:
|
|
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
|
|
directory = Path(temporary)
|
|
source = directory / "source"
|
|
if not row["staging_key"] or row["staging_deleted"]:
|
|
raise PermanentFailure("source_missing")
|
|
try:
|
|
chunks, declared_length = await storage.open_download(row["staging_key"])
|
|
except ObjectMissing:
|
|
raise PermanentFailure("source_missing") from None
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
try:
|
|
with source.open("wb") as output:
|
|
async for chunk in chunks:
|
|
size += len(chunk)
|
|
if size > settings.max_upload_bytes:
|
|
raise PermanentFailure("source_too_large")
|
|
digest.update(chunk)
|
|
output.write(chunk)
|
|
finally:
|
|
await chunks.aclose()
|
|
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
|
raise PermanentFailure("source_checksum_mismatch")
|
|
variants = await generate(source, directory)
|
|
# API never publishes directly to this final prefix. Repeating one
|
|
# v1 job writes the same bytes to the same four keys, never appends.
|
|
with source.open("rb") as body:
|
|
await storage.put(row["object_key"], body, size, row["content_type"])
|
|
prefix = row["object_key"].rsplit("/", 1)[0]
|
|
for variant in variants:
|
|
filename = variant.pop("filename")
|
|
key = f"{prefix}/{filename}"
|
|
with (directory / filename).open("rb") as body:
|
|
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
|
|
variant["object_key"] = key
|
|
await crash_if_armed("crash_after_objects_before_commit")
|
|
return variants
|
|
|
|
|
|
async def finish(row: dict, *, variants=None, code: str | None = None,
|
|
permanent: bool = False) -> bool:
|
|
async with SessionLocal() as db:
|
|
result = await db.execute(text("""
|
|
SELECT * FROM attachments WHERE id=:id AND status='processing'
|
|
AND lease_token=:token AND lease_until>now() FOR UPDATE
|
|
"""), {"id": row["id"], "token": row["lease_token"]})
|
|
current = result.mappings().first()
|
|
if current is None:
|
|
# Lease recovery or explicit operator action already owns a
|
|
# newer attempt. Never overwrite its state with an old worker.
|
|
await db.rollback()
|
|
return False
|
|
if code is not None:
|
|
await retry_or_fail(db, current, code, permanent=permanent)
|
|
else:
|
|
await db.execute(text("""
|
|
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
|
|
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
|
|
WHERE id=:id
|
|
"""), {"id": row["id"], "variants": json.dumps(variants)})
|
|
await event(db, current, "ready")
|
|
await db.commit()
|
|
return True
|
|
|
|
|
|
async def _handle(message) -> str:
|
|
try:
|
|
job = parse_envelope(message.body)
|
|
except (ValueError, TypeError, KeyError, AttributeError):
|
|
logger.warning("invalid envelope sent to durable DLQ")
|
|
await message.reject(requeue=False)
|
|
return "invalid"
|
|
row = await claim(job)
|
|
if row is None:
|
|
# A ready/failed/stale attempt is a durable no-op. A duplicate of
|
|
# active processing is covered by the DB lease recovery, so it too
|
|
# can be acknowledged without running a second concurrent decoder.
|
|
await message.ack()
|
|
return "duplicate"
|
|
outcome = "error"
|
|
try:
|
|
async with asyncio.timeout(TOTAL_TIMEOUT):
|
|
variants = await build_objects(row)
|
|
except PermanentFailure as exc:
|
|
await finish(row, code=str(exc), permanent=True)
|
|
except Exception as exc:
|
|
logger.warning("temporary processing failure attachment=%s type=%s",
|
|
row["id"], type(exc).__name__)
|
|
await finish(row, code="processing_failed")
|
|
else:
|
|
if await finish(row, variants=variants):
|
|
await crash_if_armed("crash_after_commit_before_ack")
|
|
logger.info("ready attachment=%s", row["id"])
|
|
outcome = "success"
|
|
else:
|
|
outcome = "duplicate"
|
|
# If finish failed, the exception propagates and the connection closes:
|
|
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
|
|
await message.ack()
|
|
return outcome
|
|
|
|
|
|
async def handle(message) -> None:
|
|
# Technical IDs in logs/spans only; metric dimensions stay low-cardinality.
|
|
job = {}
|
|
try:
|
|
job = parse_envelope(message.body)
|
|
except (ValueError, TypeError, KeyError, AttributeError):
|
|
pass
|
|
started = time.perf_counter()
|
|
outcome = "error"
|
|
with span("attachments process", kind=SpanKind.CONSUMER, attributes={
|
|
"messaging.system": "rabbitmq", "messaging.destination.name": "attachments",
|
|
"messaging.operation": "process", "job_type": "attachment.process.v1",
|
|
}) as active:
|
|
try:
|
|
outcome = await _handle(message)
|
|
finally:
|
|
duration = time.perf_counter() - started
|
|
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
|
|
jobs_processed.add(1, attrs)
|
|
if outcome in {"success", "error"}:
|
|
jobs_duration.record(duration, attrs)
|
|
active.set_attribute("outcome", outcome)
|
|
if outcome in {"error", "invalid"}:
|
|
active.set_status(Status(StatusCode.ERROR))
|
|
for name in ("job_id", "attachment_id", "attempt"):
|
|
if name in job:
|
|
active.set_attribute(name, job[name])
|
|
emit("job.complete", severity=logging.WARNING if outcome == "error" else logging.INFO,
|
|
**attrs, job_id=job.get("job_id"), attachment_id=job.get("attachment_id"),
|
|
attempt=job.get("attempt"), duration_seconds=duration)
|
|
|
|
|
|
async def main() -> None:
|
|
configure("messenger-worker")
|
|
install_termination_handler()
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
|
for name in ("aio_pika", "aiormq"):
|
|
logging.getLogger(name).setLevel(logging.CRITICAL)
|
|
await storage.start()
|
|
connection = None
|
|
try:
|
|
await arm_from_env("WORKER_FAULT")
|
|
while True:
|
|
try:
|
|
connection, _, queue = await broker_channel()
|
|
async with queue.iterator() as deliveries:
|
|
async for message in deliveries:
|
|
await handle(message)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except Exception as exc:
|
|
logger.warning("worker reconnect: %s", type(exc).__name__)
|
|
if connection:
|
|
await connection.close()
|
|
connection = None
|
|
await asyncio.sleep(1)
|
|
finally:
|
|
if connection:
|
|
await connection.close()
|
|
await storage.close()
|
|
await engine.dispose()
|
|
await asyncio.to_thread(shutdown)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
asyncio.run(main())
|
|
except asyncio.CancelledError:
|
|
pass # SIGTERM has run finally/SDK flush; exit cleanly.
|