"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work.""" import re from datetime import datetime from typing import Literal from uuid import UUID import redis from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator STREAM = "attachment-jobs" GROUP = "image-workers-v1" DLQ = "invalid-jobs" class Job(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) schema_version: Literal[1] job_id: str type: Literal["attachment.process.v1"] attachment_id: str pipeline_version: int = Field(ge=1) attempt: int = Field(ge=1) created_at: str traceparent: str | None = Field( default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$" ) tracestate: str | None = Field(default=None, max_length=512) @model_validator(mode="before") @classmethod def strict_carrier(cls, value): if isinstance(value, dict): if type(value.get("schema_version")) is not int: raise ValueError("Integer schema version required") if any(k in value and value[k] is None for k in ("traceparent", "tracestate")): raise ValueError("Optional carrier fields must be strings when provided") return value @field_validator("job_id", "attachment_id") @classmethod def uuid(cls, value): if str(UUID(value)) != value: raise ValueError("Canonical UUID required") return value @field_validator("created_at") @classmethod def timestamp(cls, value): if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value): raise ValueError("RFC3339 required") datetime.fromisoformat(value.replace("Z", "+00:00")) return value class Broker: def __init__(self, settings): self.client = redis.Redis.from_url( settings.broker_url, decode_responses=True, socket_timeout=2, socket_connect_timeout=1, retry_on_timeout=False, ) def group(self): try: self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True) except redis.ResponseError as exc: if "BUSYGROUP" not in str(exc): raise def publish(self, job): # XADD reply is the publisher boundary; appendfsync=always precedes that reply. return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)}) def ack(self, id): # A single consumer group only. Atomically remove completed entries, never MAXLEN. self.client.eval( "redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])", 1, STREAM, GROUP, id, ) def quarantine(self, id, reason): # Never copy arbitrary attacker-supplied fields into DLQ/logs. with self.client.pipeline(transaction=True) as tx: tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000) tx.xack(STREAM, GROUP, id) tx.xdel(STREAM, id) tx.execute() def close(self): self.client.close()