Files
mtusi2026lab5/app/pipeline.py
T

593 lines
23 KiB
Python

"""Idempotent attachment pipeline.
The effect key is (attachment_id, pipeline_version, variant). Object keys include that
identity, database rows use it as a primary key, and a completed job is a no-op.
Delivery is at least once; this module does not claim exactly-once delivery.
"""
from __future__ import annotations
import hashlib
import logging
import os
import threading
import time
import uuid
from psycopg.types.json import Jsonb
from app import runtime
from app.db import connection
from app.images import ImageRejected, decode_image, pipeline_params, render_variants
from app.outbox import add_outbox, audit, publish_dlq
log = logging.getLogger("messenger.pipeline")
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
class TransientJobError(Exception):
def __init__(self, message: str):
super().__init__(message)
self.message = message
class PermanentJobError(Exception):
def __init__(self, code: str, message: str):
super().__init__(message)
self.code = code
self.message = message
def _safe(text: str) -> str:
cleaned = " ".join(text.split())
return cleaned[:180] or "processing failed"
def _parse_message(body: dict) -> dict:
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
if not isinstance(body, dict) or set(body) - required - {"traceparent", "tracestate"}:
raise PermanentJobError("processing_failed", "invalid job message")
if any(key not in body for key in required):
raise PermanentJobError("processing_failed", "invalid job message")
if body["schema_version"] != 1 or body["type"] != "attachment.process.v1":
raise PermanentJobError("processing_failed", "invalid job message")
try:
return {
"job_id": uuid.UUID(str(body["job_id"])),
"attachment_id": uuid.UUID(str(body["attachment_id"])),
"pipeline_version": int(body["pipeline_version"]),
"attempt": int(body["attempt"]),
"created_at": str(body["created_at"]),
}
except (ValueError, TypeError) as exc:
raise PermanentJobError("processing_failed", "invalid job message") from exc
def _read_fault(conn) -> dict:
row = conn.execute("SELECT value FROM runtime_control WHERE key = 'worker_fault' FOR UPDATE").fetchone()
value = row["value"] if row else {}
return dict(value) if isinstance(value, dict) else {}
def _write_fault(conn, value: dict) -> None:
conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'worker_fault'", (Jsonb(value),))
def _consume_counted_fault() -> str | None:
"""Disarm one-shot and counted faults. Crash faults stay armed until their injection point."""
with connection() as conn:
value = _read_fault(conn)
if not value.get("armed"):
return None
mode = value.get("mode")
if mode == "transient":
remaining = int(value.get("remaining") or 1) - 1
if remaining <= 0:
_write_fault(conn, {"mode": None, "armed": False})
else:
_write_fault(conn, {"mode": "transient", "armed": True, "remaining": remaining})
return "transient"
if mode in {"permanent", "duplicate"}:
_write_fault(conn, {"mode": None, "armed": False})
return mode
if mode in {"before_effect", "after_s3", "after_commit"}:
return mode
return None
def _crash(mode: str, job_id, attachment_id) -> None:
with connection() as conn:
value = _read_fault(conn)
if value.get("armed") and value.get("mode") == mode:
_write_fault(conn, {"mode": None, "armed": False})
audit(conn, job_id, attachment_id, "fault_" + mode, {"mode": mode})
if mode in {"before_effect", "after_s3"}:
conn.execute(
"""
UPDATE jobs
SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
WHERE job_id = %s AND status = 'processing'
""",
(job_id,),
)
conn.execute(
"""
UPDATE attachments AS a
SET status = 'queued', updated_at = clock_timestamp()
FROM jobs AS j
WHERE j.job_id = %s AND a.id = j.attachment_id
AND a.pipeline_version = j.pipeline_version AND a.status = 'processing'
""",
(job_id,),
)
log.warning("fault injection exit: %s", mode)
os._exit(75)
def _load(conn, message: dict):
job = conn.execute("SELECT * FROM jobs WHERE job_id = %s FOR UPDATE", (message["job_id"],)).fetchone()
attachment = None
if job is not None:
attachment = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (job["attachment_id"],)).fetchone()
return job, attachment
def _finish_noop(conn, job, attachment, action: str, redelivered: bool) -> None:
audit(conn, job["job_id"] if job else None, attachment["id"] if attachment else None, action, {"redelivered": redelivered})
def _supersede(conn, job, attachment, redelivered: bool) -> None:
conn.execute(
"""
UPDATE jobs
SET status = 'superseded', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
WHERE job_id = %s AND status IN ('queued', 'processing')
""",
(job["job_id"],),
)
_finish_noop(conn, job, attachment, "superseded", redelivered)
def _acquire(conn, job, worker_id: str, timeout_seconds: int):
return conn.execute(
"""
UPDATE jobs
SET status = 'processing',
lease_owner = %s,
lease_until = clock_timestamp() + make_interval(secs => %s),
updated_at = clock_timestamp()
WHERE job_id = %s
AND status IN ('queued', 'processing')
AND (lease_until IS NULL OR lease_until < clock_timestamp() OR lease_owner = %s)
RETURNING job_id
""",
(worker_id, timeout_seconds, job["job_id"], worker_id),
).fetchone()
def _schedule_retry(conn, job, attachment, message: str) -> None:
settings, _store, _publisher = runtime.require()
attempt = int(job["attempt"])
if attempt >= settings.job_max_attempts:
_fail(conn, job, attachment, "retry_exhausted", "transient", "retry budget exhausted")
return
import random
delay = min(settings.job_backoff_cap_seconds, settings.job_backoff_base_seconds * (2 ** (attempt - 1)))
delay = delay * (0.5 + random.random())
next_attempt = attempt + 1
conn.execute(
"""
UPDATE jobs
SET status = 'queued', attempt = %s, next_attempt_at = clock_timestamp() + make_interval(secs => %s),
lease_until = NULL, lease_owner = NULL, error_class = 'transient', error_code = 'processing_failed',
last_error = %s, updated_at = clock_timestamp()
WHERE job_id = %s
""",
(next_attempt, delay, _safe(message), job["job_id"]),
)
conn.execute(
"""
UPDATE attachments
SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()
WHERE id = %s AND pipeline_version = %s AND status <> 'ready'
""",
(attachment["id"], job["pipeline_version"]),
)
add_outbox(conn, job["job_id"], attachment["id"], job["pipeline_version"], next_attempt, job["created_at"])
audit(conn, job["job_id"], attachment["id"], "retry", {"attempt": next_attempt, "delay_seconds": round(delay, 3)})
def _fail(conn, job, attachment, code: str, error_class: str, message: str) -> None:
conn.execute(
"""
UPDATE jobs
SET status = 'failed', error_code = %s, error_class = %s, last_error = %s,
lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
WHERE job_id = %s
""",
(code, error_class, _safe(message), job["job_id"]),
)
if attachment["ready_pipeline_version"] is not None:
# A failed newer attempt must not discard an already published result.
conn.execute(
"""
UPDATE attachments
SET status = 'ready', error_code = NULL, pipeline_version = ready_pipeline_version,
updated_at = clock_timestamp()
WHERE id = %s AND ready_pipeline_version IS NOT NULL
""",
(attachment["id"],),
)
else:
conn.execute(
"""
UPDATE attachments
SET status = 'failed', error_code = %s, updated_at = clock_timestamp()
WHERE id = %s AND pipeline_version = %s AND status <> 'ready'
""",
(code, attachment["id"], job["pipeline_version"]),
)
conn.execute(
"""
INSERT INTO failed_jobs (job_id, attachment_id, pipeline_version, attempt, error_code, error_class, safe_message)
VALUES (%s, %s, %s, %s, %s, %s, %s)
""",
(job["job_id"], attachment["id"], job["pipeline_version"], job["attempt"], code, error_class, _safe(message)),
)
audit(conn, job["job_id"], attachment["id"], "failed", {"error_code": code, "error_class": error_class})
publish_dlq(
{
"schema_version": 1,
"job_id": str(job["job_id"]),
"type": "attachment.process.v1",
"attachment_id": str(attachment["id"]),
"pipeline_version": int(job["pipeline_version"]),
"attempt": int(job["attempt"]),
"created_at": message_created(job),
}
)
def message_created(job) -> str:
from app.timeutil import iso as format_iso
value = job["created_at"]
return value if isinstance(value, str) else format_iso(value)
def _put_original(store, attachment, data: bytes) -> None:
existing = store.get(attachment["original_key"])
digest = hashlib.sha256(data).hexdigest()
if digest != attachment["sha256"]:
raise PermanentJobError("processing_failed", "checksum mismatch")
if existing is None:
store.put(attachment["original_key"], data, attachment["content_type"])
return
if hashlib.sha256(existing).hexdigest() != attachment["sha256"]:
raise TransientJobError("stored original does not match the accepted checksum")
def _store_variants(store, attachment, version: int, rendered: dict, max_pixels: int) -> list[dict]:
stored = []
for name in VARIANT_NAMES:
item = rendered[name]
key = f"variants/{attachment['id']}/v{version}/{name}"
existing = store.get(key)
if existing is None:
store.put(key, item["bytes"], item["content_type"])
blob = item["bytes"]
width, height = item["width"], item["height"]
else:
blob = existing
if hashlib.sha256(blob).hexdigest() == hashlib.sha256(item["bytes"]).hexdigest():
width, height = item["width"], item["height"]
else:
probed = decode_image(blob, max_pixels)
width, height = probed.size
stored.append(
{
"name": name,
"content_type": item["content_type"] if existing is None else _content_type_for(name, blob, item["content_type"]),
"size_bytes": len(blob),
"width": width,
"height": height,
"sha256": hashlib.sha256(blob).hexdigest(),
"object_key": key,
}
)
return stored
def _content_type_for(name: str, blob: bytes, encoded_type: str) -> str:
if blob.startswith(b"\xff\xd8"):
return "image/jpeg"
if len(blob) >= 12 and blob[:4] == b"RIFF" and blob[8:12] == b"WEBP":
return "image/webp"
return encoded_type
def _source_bytes(store, attachment) -> bytes:
data = store.get(attachment["staging_key"])
if data is None:
data = store.get(attachment["original_key"])
if data is None:
raise TransientJobError("source object is not available yet")
return data
def _commit_success(job, attachment, variants: list[dict], worker_id: str, crash_after_commit: bool) -> bool:
"""Returns True when this version became the published result."""
settings, store, _publisher = runtime.require()
with connection() as conn:
current = conn.execute("SELECT pipeline_version, ready_pipeline_version FROM attachments WHERE id = %s FOR UPDATE", (attachment["id"],)).fetchone()
for item in variants:
conn.execute(
"""
INSERT INTO attachment_variants (
attachment_id, pipeline_version, name, content_type, size_bytes, width, height, sha256, object_key
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING
""",
(
attachment["id"],
job["pipeline_version"],
item["name"],
item["content_type"],
item["size_bytes"],
item["width"],
item["height"],
item["sha256"],
item["object_key"],
),
)
updated = conn.execute(
"""
UPDATE attachments
SET status = 'ready', ready_pipeline_version = %s, error_code = NULL, updated_at = clock_timestamp()
WHERE id = %s
AND pipeline_version = %s
AND (ready_pipeline_version IS NULL OR ready_pipeline_version <= %s)
RETURNING id
""",
(job["pipeline_version"], attachment["id"], job["pipeline_version"], job["pipeline_version"]),
).fetchone()
conn.execute(
"""
UPDATE jobs
SET status = 'ready', error_code = NULL, error_class = NULL, last_error = NULL,
lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp()
WHERE job_id = %s
""",
(job["job_id"],),
)
if crash_after_commit:
_write_fault(conn, {"mode": None, "armed": False})
audit(conn, job["job_id"], attachment["id"], "fault_after_commit", {"worker": worker_id})
audit(
conn,
job["job_id"],
attachment["id"],
"committed",
{"pipeline_version": job["pipeline_version"], "published": bool(updated), "variants": len(variants)},
)
published = updated is not None and current is not None
if crash_after_commit:
os._exit(75)
if published:
try:
store.delete(attachment["staging_key"])
with connection() as conn:
conn.execute(
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s AND status = 'ready'",
(attachment["id"],),
)
except Exception:
log.warning("staging cleanup deferred for %s", attachment["id"])
return published
def _execute(message: dict, attachment: dict, worker_id: str, fault: str | None) -> None:
settings, store, _publisher = runtime.require()
if fault == "before_effect":
_crash("before_effect", message["job_id"], attachment["id"])
deadline = time.monotonic() + settings.job_timeout_seconds
data = _source_bytes(store, attachment)
if len(data) > settings.max_upload_bytes:
raise PermanentJobError("invalid_image", "object exceeds the byte limit")
if time.monotonic() > deadline:
raise TransientJobError("processing deadline exceeded")
try:
image = _decode_limited(data, settings.max_image_pixels, max(1, deadline - time.monotonic()))
params = pipeline_params(int(attachment_version(message)))
rendered = render_variants(image, params)
except ImageRejected as exc:
raise PermanentJobError("invalid_image", exc.message) from exc
if time.monotonic() > deadline:
raise TransientJobError("processing deadline exceeded")
_put_original(store, attachment, data)
variants = _store_variants(store, attachment, message["pipeline_version"], rendered, settings.max_image_pixels)
if fault == "after_s3":
_crash("after_s3", message["job_id"], attachment["id"])
_commit_success(message_as_job(message, attachment), attachment, variants, worker_id, fault == "after_commit")
def attachment_version(message: dict) -> int:
return int(message["pipeline_version"])
def message_as_job(message: dict, attachment: dict) -> dict:
return {
"job_id": message["job_id"],
"pipeline_version": message["pipeline_version"],
"attempt": message["attempt"],
"created_at": message["created_at"],
"attachment_id": attachment["id"],
}
def _decode_limited(data: bytes, max_pixels: int, timeout: float):
box: dict = {}
def run() -> None:
try:
box["image"] = decode_image(data, max_pixels)
except Exception as exc: # noqa: BLE001 - transferred to the caller
box["error"] = exc
thread = threading.Thread(target=run, daemon=True)
thread.start()
thread.join(timeout)
if thread.is_alive():
raise TransientJobError("decode timed out")
if "error" in box:
raise box["error"]
return box["image"]
def _handle_known_error(message: dict, exc: Exception) -> None:
with connection() as conn:
job, attachment = _load(conn, message)
if job is None or attachment is None or job["status"] in {"ready", "failed", "superseded"}:
return
if isinstance(exc, PermanentJobError):
_fail(conn, job, attachment, exc.code, "permanent", exc.message)
elif isinstance(exc, TransientJobError):
_schedule_retry(conn, job, attachment, exc.message)
else:
_schedule_retry(conn, job, attachment, "temporary processing failure")
def process_delivery(raw: bytes, redelivered: bool, worker_id: str) -> str:
"""Returns 'ack' or 'nack'. Ack happens only after the durable decision is committed."""
import json
try:
body = json.loads(raw.decode("utf-8"))
message = _parse_message(body)
except (UnicodeError, json.JSONDecodeError, PermanentJobError):
log.warning("discarding a job message that does not match the contract")
return "ack"
try:
fault = _consume_counted_fault()
except Exception:
log.warning("fault control unreadable", exc_info=True)
return "nack"
if fault == "transient":
_handle_known_error(message, TransientJobError("injected transient failure"))
return "ack"
if fault == "permanent":
_handle_known_error(message, PermanentJobError("invalid_image", "injected permanent failure"))
return "ack"
runs = 2 if fault == "duplicate" else 1
for _ in range(runs):
outcome = _process_once(message, redelivered, worker_id, fault if runs == 1 else None)
if outcome == "nack":
return "nack"
fault = None
return "ack"
def _process_once(message: dict, redelivered: bool, worker_id: str, fault: str | None) -> str:
settings, _store, _publisher = runtime.require()
with connection() as conn:
job, attachment = _load(conn, message)
if job is None or attachment is None:
audit(conn, message["job_id"], message["attachment_id"], "missing", {"redelivered": redelivered})
return "ack"
if job["status"] in {"ready", "superseded", "failed"}:
_finish_noop(conn, job, attachment, "noop_" + job["status"], redelivered)
return "ack"
if message["attempt"] != int(job["attempt"]):
_finish_noop(conn, job, attachment, "noop_attempt", redelivered)
return "ack"
newer_ready = (
attachment["ready_pipeline_version"] is not None
and int(attachment["ready_pipeline_version"]) > int(job["pipeline_version"])
)
newer_desired = int(attachment["pipeline_version"]) > int(job["pipeline_version"])
if newer_ready or newer_desired:
_supersede(conn, job, attachment, redelivered)
return "ack"
if _acquire(conn, job, worker_id, settings.job_timeout_seconds) is None:
return "nack"
conn.execute(
"""
UPDATE attachments
SET status = 'processing', updated_at = clock_timestamp()
WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing')
""",
(attachment["id"], job["pipeline_version"]),
)
audit(conn, job["job_id"], attachment["id"], "leased", {"worker": worker_id, "redelivered": redelivered})
try:
_execute(message, attachment, worker_id, fault)
except (PermanentJobError, TransientJobError, ImageRejected) as exc:
if isinstance(exc, ImageRejected):
exc = PermanentJobError("invalid_image", exc.message)
_handle_known_error(message, exc)
except Exception:
log.warning("unexpected pipeline error", exc_info=True)
_handle_known_error(message, TransientJobError("temporary processing failure"))
return "ack"
def recover_expired_leases() -> int:
recovered = 0
with connection() as conn:
rows = conn.execute(
"""
UPDATE jobs
SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp()
WHERE status = 'processing' AND lease_until IS NOT NULL AND lease_until < clock_timestamp()
RETURNING job_id, attachment_id, pipeline_version, attempt, created_at
"""
).fetchall()
for job in rows:
conn.execute(
"""
UPDATE attachments
SET status = 'queued', updated_at = clock_timestamp()
WHERE id = %s AND pipeline_version = %s AND status = 'processing'
""",
(job["attachment_id"], job["pipeline_version"]),
)
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], job["created_at"])
audit(conn, job["job_id"], job["attachment_id"], "lease_expired", {})
recovered += 1
if recovered:
log.info("recovered %s expired lease(s)", recovered)
return recovered
def sweep_staging() -> None:
settings, store, _publisher = runtime.require()
with connection() as conn:
ready = conn.execute(
"""
SELECT id, staging_key FROM attachments
WHERE staging_deleted_at IS NULL AND status = 'ready'
LIMIT 25
"""
).fetchall()
failed = conn.execute(
"""
SELECT id, staging_key FROM attachments
WHERE staging_deleted_at IS NULL AND status = 'failed'
AND updated_at < clock_timestamp() - make_interval(secs => %s)
LIMIT 25
""",
(settings.staging_retention_seconds,),
).fetchall()
for row in list(ready) + list(failed):
try:
store.delete(row["staging_key"])
except Exception:
log.warning("staging delete failed")
continue
with connection() as conn:
conn.execute(
"UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s",
(row["id"],),
)