Files
mtusi2026lab6/app/worker.py
T

267 lines
11 KiB
Python

"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
from __future__ import annotations
import asyncio
import hashlib
import json
import logging
from pathlib import Path
import sys
import tempfile
import time
from uuid import UUID, uuid4
from sqlalchemy import text
from app.config import settings
from app.db import SessionLocal, engine
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
arm_from_env, broker_channel, crash_if_armed,
event, parse_envelope, retry_or_fail)
from app.storage import ObjectMissing, storage
from app.telemetry import (configure, emit, install_termination_handler, jobs_duration,
jobs_processed, shutdown, span)
from opentelemetry.trace import SpanKind, Status, StatusCode
from app import advanced
from app.trace_context import carrier, parent_context
logger = logging.getLogger("messenger.worker")
class PermanentFailure(Exception):
pass
async def claim(job: dict) -> dict | None:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT * FROM attachments WHERE id=:id FOR UPDATE
"""), {"id": UUID(job["attachment_id"])})
row = result.mappings().first()
if row is None or row["job_id"] != UUID(job["job_id"]) or (
row["pipeline_version"] != job["pipeline_version"] or
row["attempt"] != job["attempt"] or row["status"] != "queued"
):
await db.rollback()
return None
# An early/manual duplicate cannot bypass the durable retry date.
result = await db.execute(text("""
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
traceparent=:traceparent,tracestate=:tracestate,
updated_at=now()
WHERE id=:id AND next_attempt_at<=now() RETURNING *
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS,
"traceparent": carrier().get("traceparent"), "tracestate": carrier().get("tracestate")})
claimed = result.mappings().first()
if claimed is None:
await db.rollback()
return None
claimed = dict(claimed)
await event(db, claimed, "processing")
await db.commit()
return claimed
async def generate(source: Path, directory: Path) -> list[dict]:
process = await asyncio.create_subprocess_exec(
sys.executable, "-m", "app.image_worker", str(source), str(directory),
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
)
try:
try:
async with asyncio.timeout(IMAGE_TIMEOUT):
code = await process.wait()
except TimeoutError:
raise PermanentFailure("image_timeout") from None
if code == 21:
raise PermanentFailure("image_resource_limit")
if code != 0:
raise PermanentFailure("invalid_image")
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
finally:
# Includes cancellation of the entire job. Native image code cannot
# continue using CPU/RAM after its parent timed out.
if process.returncode is None:
process.kill()
await process.wait()
async def build_objects(row: dict) -> list[dict]:
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
directory = Path(temporary)
source = directory / "source"
if not row["staging_key"] or row["staging_deleted"]:
raise PermanentFailure("source_missing")
try:
chunks, declared_length = await storage.open_download(row["staging_key"])
except ObjectMissing:
raise PermanentFailure("source_missing") from None
digest = hashlib.sha256()
size = 0
try:
with source.open("wb") as output:
async for chunk in chunks:
size += len(chunk)
if size > settings.max_upload_bytes:
raise PermanentFailure("source_too_large")
digest.update(chunk)
output.write(chunk)
finally:
await chunks.aclose()
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
raise PermanentFailure("source_checksum_mismatch")
with span("image.generate"):
variants = await generate(source, directory)
# API never publishes directly to this final prefix. Repeating one
# v1 job writes the same bytes to the same four keys, never appends.
with source.open("rb") as body:
await storage.put(row["object_key"], body, size, row["content_type"])
prefix = row["object_key"].rsplit("/", 1)[0]
for variant in variants:
filename = variant.pop("filename")
key = f"{prefix}/{filename}"
with (directory / filename).open("rb") as body:
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
variant["object_key"] = key
await crash_if_armed("crash_after_objects_before_commit")
return variants
async def finish(row: dict, *, variants=None, code: str | None = None,
permanent: bool = False) -> bool:
async with SessionLocal() as db:
result = await db.execute(text("""
SELECT * FROM attachments WHERE id=:id AND status='processing'
AND lease_token=:token AND lease_until>now() FOR UPDATE
"""), {"id": row["id"], "token": row["lease_token"]})
current = result.mappings().first()
if current is None:
# Lease recovery or explicit operator action already owns a
# newer attempt. Never overwrite its state with an old worker.
await db.rollback()
return False
if code is not None:
await retry_or_fail(db, current, code, permanent=permanent)
else:
await db.execute(text("""
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
WHERE id=:id
"""), {"id": row["id"], "variants": json.dumps(variants)})
await event(db, current, "ready")
await db.commit()
if code is None:
advanced.observe_ready(current)
return True
async def _handle(message) -> str:
try:
job = parse_envelope(message.body)
except (ValueError, TypeError, KeyError, AttributeError):
logger.warning("invalid envelope sent to durable DLQ")
await message.reject(requeue=False)
return "invalid"
row = await claim(job)
if row is None:
# A ready/failed/stale attempt is a durable no-op. A duplicate of
# active processing is covered by the DB lease recovery, so it too
# can be acknowledged without running a second concurrent decoder.
await message.ack()
return "duplicate"
outcome = "error"
try:
async with asyncio.timeout(TOTAL_TIMEOUT):
variants = await build_objects(row)
except PermanentFailure as exc:
await finish(row, code=str(exc), permanent=True)
except Exception as exc:
logger.warning("temporary processing failure attachment=%s type=%s",
row["id"], type(exc).__name__)
await finish(row, code="processing_failed")
else:
if await finish(row, variants=variants):
await crash_if_armed("crash_after_commit_before_ack")
logger.info("ready attachment=%s", row["id"])
outcome = "success"
else:
outcome = "duplicate"
# If finish failed, the exception propagates and the connection closes:
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
await message.ack()
return outcome
async def handle(message) -> None:
# Technical IDs in logs/spans only; metric dimensions stay low-cardinality.
job = {}
try:
job = parse_envelope(message.body)
except (ValueError, TypeError, KeyError, AttributeError):
pass
started = time.perf_counter()
outcome = "error"
with span("attachments process", kind=SpanKind.CONSUMER, parent=parent_context(job), attributes={
"messaging.system": "rabbitmq", "messaging.destination.name": "attachments",
"messaging.operation": "process", "job_type": "attachment.process.v1",
}) as active:
try:
advanced.busy = 1
outcome = await _handle(message)
finally:
advanced.busy = 0
duration = time.perf_counter() - started
attrs = {"job_type": "attachment.process.v1", "outcome": outcome}
jobs_processed.add(1, attrs)
if outcome in {"success", "error"}:
jobs_duration.record(duration, attrs)
active.set_attribute("outcome", outcome)
if outcome in {"error", "invalid"}:
active.set_status(Status(StatusCode.ERROR))
for name in ("job_id", "attachment_id", "attempt"):
if name in job:
active.set_attribute(name, job[name])
emit("job.complete", severity=logging.WARNING if outcome == "error" else logging.INFO,
**attrs, job_id=job.get("job_id"), attachment_id=job.get("attachment_id"),
attempt=job.get("attempt"), duration_seconds=duration)
async def main() -> None:
configure("messenger-worker")
install_termination_handler()
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
for name in ("aio_pika", "aiormq"):
logging.getLogger(name).setLevel(logging.CRITICAL)
await storage.start()
connection = None
try:
await arm_from_env("WORKER_FAULT")
while True:
try:
connection, _, queue = await broker_channel()
async with queue.iterator() as deliveries:
async for message in deliveries:
await handle(message)
except asyncio.CancelledError:
raise
except Exception as exc:
logger.warning("worker reconnect: %s", type(exc).__name__)
if connection:
await connection.close()
connection = None
await asyncio.sleep(1)
finally:
if connection:
await connection.close()
await storage.close()
await engine.dispose()
await asyncio.to_thread(shutdown)
if __name__ == "__main__":
try:
asyncio.run(main())
except asyncio.CancelledError:
pass # SIGTERM has run finally/SDK flush; exit cleanly.