Import committed lab5 baseline 03899d7 via git archive: API replicas, PostgreSQL, private S3, RabbitMQ, publisher, worker, migrations, and hooks. Preserve lab6 contracts and the separate telemetry stack. Static checks pass; Docker runtime integration remains pending.
217 lines
8.6 KiB
Python
217 lines
8.6 KiB
Python
"""At-least-once RabbitMQ consumer with versioned, idempotent S3 effects."""
|
|
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import random
|
|
import re
|
|
import signal
|
|
import time
|
|
from contextlib import contextmanager
|
|
from datetime import datetime
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from uuid import UUID
|
|
|
|
import pika
|
|
import psycopg
|
|
from minio.error import S3Error
|
|
from psycopg_pool import PoolTimeout
|
|
|
|
from .attachments import BUCKET, storage
|
|
from .db import pool
|
|
from .image_variants import InvalidImage, MAX_BYTES, render
|
|
from .pipeline_jobs import claim_job, complete_job, finish_attempt, recover_expired
|
|
|
|
|
|
logger = logging.getLogger(__name__)
|
|
QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process")
|
|
MAX_ATTEMPTS = int(os.environ.get("JOB_MAX_ATTEMPTS", "5"))
|
|
LEASE_SECONDS = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60"))
|
|
|
|
|
|
def parse_message(body: bytes) -> dict:
|
|
if len(body) > 4096:
|
|
raise ValueError("oversized job message")
|
|
item = json.loads(body)
|
|
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
|
if (not isinstance(item, dict) or not required <= item.keys() or
|
|
item.keys() - required - {"traceparent", "tracestate"} or
|
|
type(item["schema_version"]) is not int or item["schema_version"] != 1 or
|
|
item["type"] != "attachment.process.v1" or
|
|
type(item["pipeline_version"]) is not int or item["pipeline_version"] < 1 or
|
|
type(item["attempt"]) is not int or item["attempt"] < 1 or
|
|
not isinstance(item["job_id"], str) or
|
|
not isinstance(item["attachment_id"], str) or
|
|
not isinstance(item["created_at"], str) or
|
|
("traceparent" in item and
|
|
(not isinstance(item["traceparent"], str) or
|
|
re.fullmatch(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", item["traceparent"]) is None)) or
|
|
("tracestate" in item and
|
|
(not isinstance(item["tracestate"], str) or len(item["tracestate"]) > 512))):
|
|
raise ValueError("invalid job message")
|
|
UUID(item["job_id"])
|
|
UUID(item["attachment_id"])
|
|
created_at = datetime.fromisoformat(item["created_at"].replace("Z", "+00:00"))
|
|
if created_at.utcoffset() is None:
|
|
raise ValueError("job timestamp lacks timezone")
|
|
return item
|
|
|
|
|
|
def _source_bytes(job) -> bytes:
|
|
if not 0 < job["size_bytes"] <= MAX_BYTES:
|
|
raise InvalidImage("invalid accepted size")
|
|
# Reprocessing uses the byte-identical final original, so ready staging
|
|
# can be reclaimed without making a future pipeline version impossible.
|
|
key = job["staging_key"] if job["pipeline_version"] == 1 else job["object_key"]
|
|
response = storage().get_object(BUCKET, key)
|
|
try:
|
|
chunks = []
|
|
size = 0
|
|
digest = hashlib.sha256()
|
|
for chunk in response.stream(64 * 1024):
|
|
size += len(chunk)
|
|
if size > job["size_bytes"]:
|
|
raise InvalidImage("source exceeds accepted size")
|
|
chunks.append(chunk)
|
|
digest.update(chunk)
|
|
if size != job["size_bytes"] or digest.hexdigest() != job["sha256"]:
|
|
raise InvalidImage("source checksum mismatch")
|
|
return b"".join(chunks)
|
|
finally:
|
|
response.close()
|
|
response.release_conn()
|
|
|
|
|
|
def _write_variants(job, variants: dict) -> dict:
|
|
extension = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"}
|
|
stored = {}
|
|
for name, variant in variants.items():
|
|
key = (job["object_key"] if name == "original" else
|
|
f"variants/{job['attachment_id']}/v{job['pipeline_version']}/{name}.{extension[name]}")
|
|
data = variant["data"]
|
|
storage().put_object(
|
|
BUCKET, key, BytesIO(data), len(data), content_type=variant["content_type"]
|
|
)
|
|
stored[name] = {key_name: value for key_name, value in variant.items() if key_name != "data"}
|
|
stored[name]["object_key"] = key
|
|
return stored
|
|
|
|
|
|
def _delay(attempt: int) -> float:
|
|
base = min(30, 2 ** min(attempt - 1, 5))
|
|
return base + random.uniform(0, base / 2)
|
|
|
|
|
|
@contextmanager
|
|
def _deadline(seconds: int):
|
|
def expired(_signal, _frame):
|
|
raise TimeoutError("image job exceeded worker deadline")
|
|
|
|
previous = signal.signal(signal.SIGALRM, expired)
|
|
signal.setitimer(signal.ITIMER_REAL, seconds)
|
|
try:
|
|
yield
|
|
finally:
|
|
signal.setitimer(signal.ITIMER_REAL, 0)
|
|
signal.signal(signal.SIGALRM, previous)
|
|
|
|
|
|
def process_message(item: dict) -> str:
|
|
with pool().connection() as conn:
|
|
job = claim_job(conn, UUID(item["job_id"]), item["attempt"], lease_seconds=LEASE_SECONDS)
|
|
if job is None:
|
|
return "duplicate_or_stale"
|
|
token = job["lease_token"]
|
|
try:
|
|
with _deadline(LEASE_SECONDS):
|
|
data = _source_bytes(job)
|
|
variants = render(data, job["content_type"])
|
|
stored = _write_variants(job, variants)
|
|
with pool().connection() as conn:
|
|
current = complete_job(conn, job["id"], token, stored)
|
|
if current and job["staging_key"]:
|
|
try:
|
|
storage().remove_object(BUCKET, job["staging_key"])
|
|
except Exception as exc:
|
|
# Periodic cleanup recovers this post-COMMIT gap.
|
|
logger.warning("Staging cleanup deferred for %s (%s)", job["id"], type(exc).__name__)
|
|
return "ready"
|
|
except psycopg.Error:
|
|
raise
|
|
except Exception as exc:
|
|
if isinstance(exc, InvalidImage):
|
|
code, permanent = "invalid_image", True
|
|
elif isinstance(exc, TimeoutError):
|
|
code, permanent = "worker_timeout", False
|
|
elif isinstance(exc, S3Error) or isinstance(exc, OSError):
|
|
code, permanent = "storage_unavailable", False
|
|
elif isinstance(exc, ValueError):
|
|
code, permanent = "processing_failed", True
|
|
else:
|
|
code, permanent = "processing_failed", False
|
|
logger.warning("Job %s attempt %s: %s", job["id"], job["attempt"], code)
|
|
with pool().connection() as conn:
|
|
try:
|
|
return finish_attempt(
|
|
conn, job["id"], token, max_attempts=MAX_ATTEMPTS,
|
|
delay_seconds=_delay(job["attempt"]), error_code=code,
|
|
permanent=permanent,
|
|
)
|
|
except ValueError:
|
|
# A lease recovery already owns the next attempt.
|
|
return "stale_lease"
|
|
|
|
|
|
def main() -> None:
|
|
logging.basicConfig(level=logging.INFO)
|
|
if MAX_ATTEMPTS < 1 or LEASE_SECONDS < 1:
|
|
raise SystemExit("invalid worker retry settings")
|
|
url = Path(os.environ["BROKER_URL_FILE"]).read_text(encoding="utf-8").strip()
|
|
params = pika.URLParameters(url)
|
|
params.heartbeat = max(120, LEASE_SECONDS * 2)
|
|
params.socket_timeout = 5
|
|
params.blocked_connection_timeout = 10
|
|
while True:
|
|
broker = None
|
|
try:
|
|
broker = pika.BlockingConnection(params)
|
|
channel = broker.channel()
|
|
channel.queue_declare(queue=QUEUE, passive=True)
|
|
channel.basic_qos(prefetch_count=1)
|
|
last_recovery = 0.0
|
|
while broker.is_open:
|
|
if time.monotonic() - last_recovery >= 5:
|
|
with pool().connection() as conn:
|
|
recover_expired(conn, max_attempts=MAX_ATTEMPTS)
|
|
last_recovery = time.monotonic()
|
|
method, _, body = channel.basic_get(queue=QUEUE, auto_ack=False)
|
|
if method is None:
|
|
broker.process_data_events(time_limit=1)
|
|
continue
|
|
try:
|
|
item = parse_message(body)
|
|
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
|
logger.warning("Rejecting invalid job message")
|
|
channel.basic_reject(method.delivery_tag, requeue=False)
|
|
continue
|
|
try:
|
|
result = process_message(item)
|
|
channel.basic_ack(method.delivery_tag)
|
|
logger.info("Job %s: %s", item["job_id"], result)
|
|
except (psycopg.Error, PoolTimeout, OSError) as exc:
|
|
logger.warning("Job %s database unavailable (%s)", item["job_id"], type(exc).__name__)
|
|
channel.basic_nack(method.delivery_tag, requeue=True)
|
|
time.sleep(2)
|
|
except (pika.exceptions.AMQPError, psycopg.Error, PoolTimeout, OSError) as exc:
|
|
logger.warning("Worker reconnecting after %s", type(exc).__name__)
|
|
time.sleep(2)
|
|
finally:
|
|
if broker is not None and broker.is_open:
|
|
broker.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|