Files
mtusi2026lab5/src/messenger/broker.py
T

97 lines
3.1 KiB
Python

"""Dedicated durable Valkey Stream, one consumer group; never evict/trim pending work."""
import re
from datetime import datetime
from typing import Literal
from uuid import UUID
import redis
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
STREAM = "attachment-jobs"
GROUP = "image-workers-v1"
DLQ = "invalid-jobs"
class Job(BaseModel):
model_config = ConfigDict(extra="forbid", strict=True)
schema_version: Literal[1]
job_id: str
type: Literal["attachment.process.v1"]
attachment_id: str
pipeline_version: int = Field(ge=1)
attempt: int = Field(ge=1)
created_at: str
traceparent: str | None = Field(
default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
)
tracestate: str | None = Field(default=None, max_length=512)
@model_validator(mode="before")
@classmethod
def strict_carrier(cls, value):
if isinstance(value, dict):
if type(value.get("schema_version")) is not int:
raise ValueError("Integer schema version required")
if any(k in value and value[k] is None for k in ("traceparent", "tracestate")):
raise ValueError("Optional carrier fields must be strings when provided")
return value
@field_validator("job_id", "attachment_id")
@classmethod
def uuid(cls, value):
if str(UUID(value)) != value:
raise ValueError("Canonical UUID required")
return value
@field_validator("created_at")
@classmethod
def timestamp(cls, value):
if not re.fullmatch(r"\d{4}-\d\d-\d\dT.+(?:Z|[+-]\d\d:\d\d)", value):
raise ValueError("RFC3339 required")
datetime.fromisoformat(value.replace("Z", "+00:00"))
return value
class Broker:
def __init__(self, settings):
self.client = redis.Redis.from_url(
settings.broker_url,
decode_responses=True,
socket_timeout=2,
socket_connect_timeout=1,
retry_on_timeout=False,
)
def group(self):
try:
self.client.xgroup_create(STREAM, GROUP, id="0", mkstream=True)
except redis.ResponseError as exc:
if "BUSYGROUP" not in str(exc):
raise
def publish(self, job):
# XADD reply is the publisher boundary; appendfsync=always precedes that reply.
return self.client.xadd(STREAM, {"job": job.model_dump_json(exclude_none=True)})
def ack(self, id):
# A single consumer group only. Atomically remove completed entries, never MAXLEN.
self.client.eval(
"redis.call('XACK',KEYS[1],ARGV[1],ARGV[2]); return redis.call('XDEL',KEYS[1],ARGV[2])",
1,
STREAM,
GROUP,
id,
)
def quarantine(self, id, reason):
# Never copy arbitrary attacker-supplied fields into DLQ/logs.
with self.client.pipeline(transaction=True) as tx:
tx.xadd(DLQ, {"stream_id": id, "reason": reason}, maxlen=1000)
tx.xack(STREAM, GROUP, id)
tx.xdel(STREAM, id)
tx.execute()
def close(self):
self.client.close()